From 762e2f4f745974e5742cd7bdd71f7ab4ec3bfc51 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Fri, 28 Sep 2018 18:03:45 +0300 Subject: [PATCH 001/403] IGNITE-9693 Scale up wal compression workers to increase performance - Fixes #4831. Signed-off-by: Ivan Rakov --- .../apache/ignite/IgniteSystemProperties.java | 5 + .../wal/IgniteWriteAheadLogManager.java | 3 +- .../GridCacheDatabaseSharedManager.java | 11 +- .../wal/FileWriteAheadLogManager.java | 209 +++++++++++------- .../FsyncModeFileWriteAheadLogManager.java | 7 +- .../persistence/wal/aware/SegmentAware.java | 28 ++- .../wal/aware/SegmentCompressStorage.java | 80 +++++-- ...sReserveWalSegmentsWithCompactionTest.java | 34 +++ .../persistence/pagemem/NoOpWALManager.java | 2 +- .../wal/aware/SegmentAwareTest.java | 90 +++++--- .../testsuites/IgnitePdsTestSuite2.java | 2 + 11 files changed, 325 insertions(+), 146 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 4ecffbe24ee04..892689c0c1a86 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -900,6 +900,11 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE = "IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE"; + /** + * Count of WAL compressor worker threads. Default value is 4. + */ + public static final String IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT = "IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT"; + /** * Whenever read load balancing is enabled, that means 'get' requests will be distributed between primary and backup * nodes if it is possible and {@link CacheConfiguration#readFromBackup} is {@code true}. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java index 12fd3e94bd0fe..4ffa347985e59 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java @@ -86,9 +86,8 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni * Invoke this method to reserve WAL history since provided pointer and prevent it's deletion. * * @param start WAL pointer. - * @throws IgniteException If failed to reserve. */ - public boolean reserve(WALPointer start) throws IgniteCheckedException; + public boolean reserve(WALPointer start); /** * Invoke this method to release WAL history since provided pointer that was previously reserved. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 14f8fe6a96e98..aff2b426357c7 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -1758,16 +1758,7 @@ private Map> partitionsApplicableForWalRebalance() { if (ptr == null) return false; - boolean reserved; - - try { - reserved = cctx.wal().reserve(ptr); - } - catch (IgniteCheckedException e) { - U.error(log, "Error while trying to reserve history", e); - - reserved = false; - } + boolean reserved = cctx.wal().reserve(ptr); if (reserved) reservedForPreloading.put(new T2<>(grpId, partId), new T2<>(cntr, ptr)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index eca0cb6c6a64a..86b27c19bcfb4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -134,6 +134,7 @@ import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SEGMENT_SYNC_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION; @@ -257,6 +258,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private static final double THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE = IgniteSystemProperties.getDouble(IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE, 0.5); + /** + * Number of WAL compressor worker threads. + */ + private final int WAL_COMPRESSOR_WORKER_THREAD_CNT = + IgniteSystemProperties.getInteger(IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT, 4); + /** */ private final boolean alwaysWriteFullPages; @@ -415,6 +422,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { evt = ctx.event(); failureProcessor = ctx.failure(); + segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled()); } /** @@ -472,7 +480,7 @@ public void setFileIOFactory(FileIOFactory ioFactory) { IgniteBiTuple tup = scanMinMaxArchiveIndices(); - segmentAware = new SegmentAware(dsCfg.getWalSegments()); + segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled()); segmentAware.lastTruncatedArchiveIdx(tup == null ? -1 : tup.get1() - 1); @@ -487,7 +495,8 @@ public void setFileIOFactory(FileIOFactory ioFactory) { segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx); if (dsCfg.isWalCompactionEnabled()) { - compressor = new FileCompressor(); + if (compressor == null) + compressor = new FileCompressor(log); if (decompressor == null) { // Preventing of two file-decompressor thread instantiations. decompressor = new FileDecompressor(log); @@ -896,7 +905,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { } /** {@inheritDoc} */ - @Override public boolean reserve(WALPointer start) throws IgniteCheckedException { + @Override public boolean reserve(WALPointer start) { assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start; if (mode == WALMode.NONE) @@ -1006,7 +1015,7 @@ private boolean segmentReservedOrLocked(long absIdx) { /** {@inheritDoc} */ @Override public void notchLastCheckpointPtr(WALPointer ptr) { if (compressor != null) - compressor.keepUncompressedIdxFrom(((FileWALPointer)ptr).index()); + segmentAware.keepUncompressedIdxFrom(((FileWALPointer)ptr).index()); } /** {@inheritDoc} */ @@ -1911,16 +1920,13 @@ private void allocateRemainingFiles() throws StorageException { * Responsible for compressing WAL archive segments. * Also responsible for deleting raw copies of already compressed WAL archive segments if they are not reserved. */ - private class FileCompressor extends Thread { - /** Current thread stopping advice. */ - private volatile boolean stopped; - - /** All segments prior to this (inclusive) can be compressed. */ - private volatile long minUncompressedIdxToKeep = -1L; + private class FileCompressor extends FileCompressorWorker { + /** Workers queue. */ + List workers = new ArrayList<>(); /** */ - FileCompressor() { - super("wal-file-compressor%" + cctx.igniteInstanceName()); + FileCompressor(IgniteLogger log) { + super(0, log); } /** */ @@ -1928,7 +1934,7 @@ private void init() { File[] toDel = walArchiveDir.listFiles(WAL_SEGMENT_TEMP_FILE_COMPACTED_FILTER); for (File f : toDel) { - if (stopped) + if (isCancelled()) return; f.delete(); @@ -1937,82 +1943,118 @@ private void init() { FileDescriptor[] alreadyCompressed = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_COMPACTED_FILTER)); if (alreadyCompressed.length > 0) - segmentAware.lastCompressedIdx(alreadyCompressed[alreadyCompressed.length - 1].idx()); + segmentAware.onSegmentCompressed(alreadyCompressed[alreadyCompressed.length - 1].idx()); + + for (int i = 1; i < calculateThreadCount(); i++) { + FileCompressorWorker worker = new FileCompressorWorker(i, log); + + worker.start(); + + workers.add(worker); + } } /** - * @param idx Minimum raw segment index that should be preserved from deletion. + * Calculate optimal additional compressor worker threads count. If quarter of proc threads greater + * than WAL_COMPRESSOR_WORKER_THREAD_CNT, use this value. Otherwise, reduce number of threads. + * + * @return Optimal number of compressor threads. */ - void keepUncompressedIdxFrom(long idx) { - minUncompressedIdxToKeep = idx; + private int calculateThreadCount() { + int procNum = Runtime.getRuntime().availableProcessors(); + + // If quarter of proc threads greater than WAL_COMPRESSOR_WORKER_THREAD_CNT, + // use this value. Otherwise, reduce number of threads. + if (procNum >> 2 >= WAL_COMPRESSOR_WORKER_THREAD_CNT) + return WAL_COMPRESSOR_WORKER_THREAD_CNT; + else + return procNum >> 2; } - /** - * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation. - * Waits if there's no segment to archive right now. - */ - private long tryReserveNextSegmentOrWait() throws IgniteCheckedException { - long segmentToCompress = segmentAware.waitNextSegmentToCompress(); - boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0)); + /** {@inheritDoc} */ + @Override public void body() throws InterruptedException, IgniteInterruptedCheckedException{ + init(); - return reserved ? segmentToCompress : -1; + super.body0(); } /** - * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves. + * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown. */ - private void deleteObsoleteRawSegments() { - FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER)); + private void shutdown() throws IgniteInterruptedCheckedException { + synchronized (this) { + for (FileCompressorWorker worker: workers) + U.cancel(worker); - Set indices = new HashSet<>(); - Set duplicateIndices = new HashSet<>(); + for (FileCompressorWorker worker: workers) + U.join(worker); - for (FileDescriptor desc : descs) { - if (!indices.add(desc.idx)) - duplicateIndices.add(desc.idx); + U.cancel(this); } - for (FileDescriptor desc : descs) { - if (desc.isCompressed()) - continue; + U.join(this); + } + } - // Do not delete reserved or locked segment and any segment after it. - if (segmentReservedOrLocked(desc.idx)) - return; + /** */ + private class FileCompressorWorker extends GridWorker { + /** */ + private Thread thread; - if (desc.idx < minUncompressedIdxToKeep && duplicateIndices.contains(desc.idx)) { - if (!desc.file.delete()) - U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " + - desc.file.getAbsolutePath() + ", exists: " + desc.file.exists()); - } - } + /** */ + FileCompressorWorker(int idx, IgniteLogger log) { + super(cctx.igniteInstanceName(), "wal-file-compressor-%" + cctx.igniteInstanceName() + "%-" + idx, log); + } + + /** */ + void start() { + thread = new IgniteThread(this); + + thread.start(); + } + + /** + * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation. + * Waits if there's no segment to archive right now. + */ + private long tryReserveNextSegmentOrWait() throws IgniteInterruptedCheckedException{ + long segmentToCompress = segmentAware.waitNextSegmentToCompress(); + + boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0)); + + return reserved ? segmentToCompress : -1; } /** {@inheritDoc} */ - @Override public void run() { - init(); + @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException { + body0(); + } - while (!Thread.currentThread().isInterrupted() && !stopped) { - long currReservedSegment = -1; + /** */ + private void body0() { + while (!isCancelled()) { + long segIdx = -1L; try { - deleteObsoleteRawSegments(); + segIdx = tryReserveNextSegmentOrWait(); - currReservedSegment = tryReserveNextSegmentOrWait(); - if (currReservedSegment == -1) + if (segIdx <= segmentAware.lastCompressedIdx()) continue; - File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) - + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX); + deleteObsoleteRawSegments(); + + File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(segIdx) + + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX); - File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + FilePageStoreManager.ZIP_SUFFIX); + File zip = new File(walArchiveDir, FileDescriptor.fileName(segIdx) + FilePageStoreManager.ZIP_SUFFIX); + + File raw = new File(walArchiveDir, FileDescriptor.fileName(segIdx)); - File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)); if (!Files.exists(raw.toPath())) throw new IgniteCheckedException("WAL archive segment is missing: " + raw); - compressSegmentToFile(currReservedSegment, raw, tmpZip); + compressSegmentToFile(segIdx, raw, tmpZip); Files.move(tmpZip.toPath(), zip.toPath()); @@ -2023,27 +2065,27 @@ private void deleteObsoleteRawSegments() { if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED)) { evt.record(new WalSegmentCompactedEvent( - cctx.discovery().localNode(), - currReservedSegment, - zip.getAbsoluteFile()) + cctx.localNode(), + segIdx, + zip.getAbsoluteFile()) ); } } - segmentAware.lastCompressedIdx(currReservedSegment); + segmentAware.onSegmentCompressed(segIdx); } catch (IgniteInterruptedCheckedException ignore) { Thread.currentThread().interrupt(); } catch (IgniteCheckedException | IOException e) { - U.error(log, "Compression of WAL segment [idx=" + currReservedSegment + - "] was skipped due to unexpected error", e); + U.error(log, "Compression of WAL segment [idx=" + segIdx + + "] was skipped due to unexpected error", e); - segmentAware.lastCompressedIdx(currReservedSegment); + segmentAware.onSegmentCompressed(segIdx); } finally { - if (currReservedSegment != -1) - release(new FileWALPointer(currReservedSegment, 0, 0)); + if (segIdx != -1L) + release(new FileWALPointer(segIdx, 0, 0)); } } } @@ -2054,7 +2096,7 @@ private void deleteObsoleteRawSegments() { * @param zip Zip file. */ private void compressSegmentToFile(long nextSegment, File raw, File zip) - throws IOException, IgniteCheckedException { + throws IOException, IgniteCheckedException { int segmentSerializerVer; try (FileIO fileIO = ioFactory.create(raw)) { @@ -2084,7 +2126,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip) }; try (SingleSegmentLogicalRecordsIterator iter = new SingleSegmentLogicalRecordsIterator( - log, cctx, ioFactory, BUF_SIZE, nextSegment, walArchiveDir, appendToZipC)) { + log, cctx, ioFactory, BUF_SIZE, nextSegment, walArchiveDir, appendToZipC)) { while (iter.hasNextX()) iter.nextX(); @@ -2103,7 +2145,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip) * @param ser Record Serializer. */ @NotNull private ByteBuffer prepareSwitchSegmentRecordBuffer(long nextSegment, RecordSerializer ser) - throws IgniteCheckedException { + throws IgniteCheckedException { SwitchSegmentRecord switchRecord = new SwitchSegmentRecord(); int switchRecordSize = ser.size(switchRecord); @@ -2118,16 +2160,33 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip) } /** - * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown. + * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves. */ - private void shutdown() throws IgniteInterruptedCheckedException { - synchronized (this) { - stopped = true; + private void deleteObsoleteRawSegments() { + FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER)); - notifyAll(); + Set indices = new HashSet<>(); + Set duplicateIndices = new HashSet<>(); + + for (FileDescriptor desc : descs) { + if (!indices.add(desc.idx)) + duplicateIndices.add(desc.idx); } - U.join(this); + for (FileDescriptor desc : descs) { + if (desc.isCompressed()) + continue; + + // Do not delete reserved or locked segment and any segment after it. + if (segmentReservedOrLocked(desc.idx)) + return; + + if (desc.idx < segmentAware.keepUncompressedIdxFrom() && duplicateIndices.contains(desc.idx)) { + if (desc.file.exists() && !desc.file.delete()) + U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " + + desc.file.getAbsolutePath() + ", exists: " + desc.file.exists()); + } + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index df8f4de0c888d..1c0325e46cd8a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -783,7 +783,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { } /** {@inheritDoc} */ - @Override public boolean reserve(WALPointer start) throws IgniteCheckedException { + @Override public boolean reserve(WALPointer start) { assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start; if (mode == WALMode.NONE) @@ -791,8 +791,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { FileArchiver archiver0 = archiver; - if (archiver0 == null) - throw new IgniteCheckedException("Could not reserve WAL segment: archiver == null"); + assert archiver0 != null : "Could not reserve WAL segment: archiver == null"; archiver0.reserve(((FileWALPointer)start).index()); @@ -1912,7 +1911,7 @@ synchronized void onNextSegmentArchived() { * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation. * Waits if there's no segment to archive right now. */ - private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCheckedException { + private long tryReserveNextSegmentOrWait() throws InterruptedException { long segmentToCompress = lastCompressedIdx + 1; synchronized (this) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java index 3379b74cf5780..6ba03991e58a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java @@ -36,15 +36,17 @@ public class SegmentAware { /** Manages last archived index, emulates archivation in no-archiver mode. */ private final SegmentArchivedStorage segmentArchivedStorage = buildArchivedStorage(segmentLockStorage); /** Storage of actual information about current index of compressed segments. */ - private final SegmentCompressStorage segmentCompressStorage = buildCompressStorage(segmentArchivedStorage); + private final SegmentCompressStorage segmentCompressStorage; /** Storage of absolute current segment index. */ private final SegmentCurrentStateStorage segmentCurrStateStorage; /** * @param walSegmentsCnt Total WAL segments count. + * @param compactionEnabled Is wal compaction enabled. */ - public SegmentAware(int walSegmentsCnt) { + public SegmentAware(int walSegmentsCnt, boolean compactionEnabled) { segmentCurrStateStorage = buildCurrentStateStorage(walSegmentsCnt, segmentArchivedStorage); + segmentCompressStorage = buildCompressStorage(segmentArchivedStorage, compactionEnabled); } /** @@ -108,12 +110,12 @@ public long waitNextSegmentToCompress() throws IgniteInterruptedCheckedException } /** - * Force set last compressed segment. + * Callback after segment compression finish. * - * @param lastCompressedIdx Segment which was last compressed. + * @param compressedIdx Index of compressed segment. */ - public void lastCompressedIdx(long lastCompressedIdx) { - segmentCompressStorage.lastCompressedIdx(lastCompressedIdx); + public void onSegmentCompressed(long compressedIdx) { + segmentCompressStorage.onSegmentCompressed(compressedIdx); } /** @@ -123,6 +125,20 @@ public long lastCompressedIdx() { return segmentCompressStorage.lastCompressedIdx(); } + /** + * @param idx Minimum raw segment index that should be preserved from deletion. + */ + public void keepUncompressedIdxFrom(long idx) { + segmentCompressStorage.keepUncompressedIdxFrom(idx); + } + + /** + * @return Minimum raw segment index that should be preserved from deletion. + */ + public long keepUncompressedIdxFrom() { + return segmentCompressStorage.keepUncompressedIdxFrom(); + } + /** * Update current WAL index. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java index 30c9a2d50d363..174fb46bb6070 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java @@ -18,6 +18,10 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.aware; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; /** * Storage of actual information about current index of compressed segments. @@ -25,25 +29,50 @@ public class SegmentCompressStorage { /** Flag of interrupt waiting on this object. */ private volatile boolean interrupted; + /** Manages last archived index, emulates archivation in no-archiver mode. */ private final SegmentArchivedStorage segmentArchivedStorage; + + /** If WAL compaction enabled. */ + private final boolean compactionEnabled; + /** Last successfully compressed segment. */ private volatile long lastCompressedIdx = -1L; + /** Last enqueued to compress segment. */ + private long lastEnqueuedToCompressIdx = -1L; + + /** Segments to compress queue. */ + private final Queue segmentsToCompress = new ArrayDeque<>(); + + /** List of currently compressing segments. */ + private final List compressingSegments = new ArrayList<>(); + + /** Compressed segment with maximal index. */ + private long lastMaxCompressedIdx = -1L; + + /** Min uncompressed index to keep. */ + private volatile long minUncompressedIdxToKeep = -1L; + /** * @param segmentArchivedStorage Storage of last archived segment. + * @param compactionEnabled If WAL compaction enabled. */ - private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage) { + private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage, boolean compactionEnabled) { this.segmentArchivedStorage = segmentArchivedStorage; + this.compactionEnabled = compactionEnabled; + this.segmentArchivedStorage.addObserver(this::onSegmentArchived); } /** * @param segmentArchivedStorage Storage of last archived segment. + * @param compactionEnabled If WAL compaction enabled. */ - static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage) { - SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage); + static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage, + boolean compactionEnabled) { + SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage, compactionEnabled); segmentArchivedStorage.addObserver(storage::onSegmentArchived); @@ -51,12 +80,20 @@ static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmen } /** - * Force set last compressed segment. + * Callback after segment compression finish. * - * @param lastCompressedIdx Segment which was last compressed. + * @param compressedIdx Index of compressed segment. */ - void lastCompressedIdx(long lastCompressedIdx) { - this.lastCompressedIdx = lastCompressedIdx; + synchronized void onSegmentCompressed(long compressedIdx) { + if (compressedIdx > lastMaxCompressedIdx) + lastMaxCompressedIdx = compressedIdx; + + compressingSegments.remove(compressedIdx); + + if (!compressingSegments.isEmpty()) + this.lastCompressedIdx = Math.min(lastMaxCompressedIdx, compressingSegments.get(0) - 1); + else + this.lastCompressedIdx = lastMaxCompressedIdx; } /** @@ -71,13 +108,8 @@ long lastCompressedIdx() { * there's no segment to archive right now. */ synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedException { - long segmentToCompress = lastCompressedIdx + 1; - try { - while ( - segmentToCompress > segmentArchivedStorage.lastArchivedAbsoluteIndex() - && !interrupted - ) + while (segmentsToCompress.peek() == null && !interrupted) wait(); } catch (InterruptedException e) { @@ -86,7 +118,11 @@ synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedE checkInterrupted(); - return segmentToCompress; + Long idx = segmentsToCompress.poll(); + + compressingSegments.add(idx); + + return idx == null ? -1L : idx; } /** @@ -110,7 +146,23 @@ private void checkInterrupted() throws IgniteInterruptedCheckedException { * Callback for waking up compressor when new segment is archived. */ private synchronized void onSegmentArchived(long lastAbsArchivedIdx) { + while (lastEnqueuedToCompressIdx < lastAbsArchivedIdx && compactionEnabled) + segmentsToCompress.add(++lastEnqueuedToCompressIdx); + notifyAll(); } + /** + * @param idx Minimum raw segment index that should be preserved from deletion. + */ + void keepUncompressedIdxFrom(long idx) { + minUncompressedIdxToKeep = idx; + } + + /** + * @return Minimum raw segment index that should be preserved from deletion. + */ + long keepUncompressedIdxFrom() { + return minUncompressedIdxToKeep; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java new file mode 100644 index 0000000000000..bc34f2905b61c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import org.apache.ignite.configuration.IgniteConfiguration; + +/** + * + */ +public class IgnitePdsReserveWalSegmentsWithCompactionTest extends IgnitePdsReserveWalSegmentsTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + cfg.getDataStorageConfiguration().setWalCompactionEnabled(true); + + return cfg; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java index 811a231524522..df894190c3e00 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java @@ -68,7 +68,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { } /** {@inheritDoc} */ - @Override public boolean reserve(WALPointer start) throws IgniteCheckedException { + @Override public boolean reserve(WALPointer start) { return false; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java index 82876845541ed..7840b0979e06c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java @@ -31,13 +31,12 @@ * Test for {@link SegmentAware}. */ public class SegmentAwareTest extends TestCase { - /** * Waiting finished when work segment is set. */ public void testFinishAwaitSegment_WhenExactWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5)); @@ -53,7 +52,7 @@ public void testFinishAwaitSegment_WhenExactWaitingSegmentWasSet() throws Ignite */ public void testFinishAwaitSegment_WhenGreaterThanWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5)); @@ -69,7 +68,7 @@ public void testFinishAwaitSegment_WhenGreaterThanWaitingSegmentWasSet() throws */ public void testFinishAwaitSegment_WhenNextSegmentEqualToWaitingOne() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5)); @@ -91,7 +90,7 @@ public void testFinishAwaitSegment_WhenNextSegmentEqualToWaitingOne() throws Ign */ public void testFinishAwaitSegment_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5)); @@ -107,7 +106,7 @@ public void testFinishAwaitSegment_WhenInterruptWasCall() throws IgniteCheckedEx */ public void testFinishWaitSegmentForArchive_WhenWorkSegmentIncremented() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.curAbsWalIdx(5); aware.setLastArchivedAbsoluteIndex(4); @@ -126,7 +125,7 @@ public void testFinishWaitSegmentForArchive_WhenWorkSegmentIncremented() throws */ public void testFinishWaitSegmentForArchive_WhenWorkSegmentGreaterValue() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.curAbsWalIdx(5); aware.setLastArchivedAbsoluteIndex(4); @@ -145,7 +144,7 @@ public void testFinishWaitSegmentForArchive_WhenWorkSegmentGreaterValue() throws */ public void testFinishWaitSegmentForArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.curAbsWalIdx(5); aware.setLastArchivedAbsoluteIndex(4); @@ -164,7 +163,7 @@ public void testFinishWaitSegmentForArchive_WhenInterruptWasCall() throws Ignite */ public void testCorrectCalculateNextSegmentIndex() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.curAbsWalIdx(5); @@ -180,7 +179,7 @@ public void testCorrectCalculateNextSegmentIndex() throws IgniteCheckedException */ public void testFinishWaitNextAbsoluteIndex_WhenMarkAsArchivedFirstSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(2); + SegmentAware aware = new SegmentAware(2, false); aware.curAbsWalIdx(1); aware.setLastArchivedAbsoluteIndex(-1); @@ -199,7 +198,7 @@ public void testFinishWaitNextAbsoluteIndex_WhenMarkAsArchivedFirstSegment() thr */ public void testFinishWaitNextAbsoluteIndex_WhenSetToArchivedFirst() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(2); + SegmentAware aware = new SegmentAware(2, false); aware.curAbsWalIdx(1); aware.setLastArchivedAbsoluteIndex(-1); @@ -218,7 +217,7 @@ public void testFinishWaitNextAbsoluteIndex_WhenSetToArchivedFirst() throws Igni */ public void testFinishWaitNextAbsoluteIndex_WhenOnlyForceInterruptWasCall() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(2); + SegmentAware aware = new SegmentAware(2, false); aware.curAbsWalIdx(2); aware.setLastArchivedAbsoluteIndex(-1); @@ -243,7 +242,7 @@ public void testFinishWaitNextAbsoluteIndex_WhenOnlyForceInterruptWasCall() thro */ public void testFinishSegmentArchived_WhenSetExactWaitingSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5)); @@ -257,9 +256,9 @@ public void testFinishSegmentArchived_WhenSetExactWaitingSegment() throws Ignite /** * Waiting finished when segment archived. */ - public void testFinishSegmentArchived_WhenMarkExactWatingSegment() throws IgniteCheckedException, InterruptedException { + public void testFinishSegmentArchived_WhenMarkExactWaitingSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5)); @@ -273,9 +272,9 @@ public void testFinishSegmentArchived_WhenMarkExactWatingSegment() throws Ignite /** * Waiting finished when segment archived. */ - public void testFinishSegmentArchived_WhenSetGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException { + public void testFinishSegmentArchived_WhenSetGreaterThanWaitingSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5)); @@ -289,9 +288,9 @@ public void testFinishSegmentArchived_WhenSetGreaterThanWatingSegment() throws I /** * Waiting finished when segment archived. */ - public void testFinishSegmentArchived_WhenMarkGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException { + public void testFinishSegmentArchived_WhenMarkGreaterThanWaitingSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5)); @@ -307,7 +306,7 @@ public void testFinishSegmentArchived_WhenMarkGreaterThanWatingSegment() throws */ public void testFinishSegmentArchived_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.curAbsWalIdx(5); aware.setLastArchivedAbsoluteIndex(4); @@ -326,7 +325,7 @@ public void testFinishSegmentArchived_WhenInterruptWasCall() throws IgniteChecke */ public void testMarkAsMovedToArchive_WhenReleaseLockedSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.checkCanReadArchiveOrReserveWorkSegment(5); @@ -344,7 +343,7 @@ public void testMarkAsMovedToArchive_WhenReleaseLockedSegment() throws IgniteChe */ public void testMarkAsMovedToArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.checkCanReadArchiveOrReserveWorkSegment(5); IgniteInternalFuture future = awaitThread(() -> aware.markAsMovedToArchive(5)); @@ -364,9 +363,9 @@ public void testMarkAsMovedToArchive_WhenInterruptWasCall() throws IgniteChecked */ public void testFinishWaitSegmentToCompress_WhenSetLastArchivedSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, true); - aware.lastCompressedIdx(5); + aware.onSegmentCompressed(5); IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress); @@ -382,9 +381,9 @@ public void testFinishWaitSegmentToCompress_WhenSetLastArchivedSegment() throws */ public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, true); - aware.lastCompressedIdx(5); + aware.onSegmentCompressed(5); IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress); @@ -400,9 +399,9 @@ public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws */ public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, true); - aware.lastCompressedIdx(5); + aware.onSegmentCompressed(5); aware.setLastArchivedAbsoluteIndex(6); aware.lastTruncatedArchiveIdx(7); @@ -418,8 +417,8 @@ public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedExcept */ public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); - aware.lastCompressedIdx(5); + SegmentAware aware = new SegmentAware(10, true); + aware.onSegmentCompressed(5); IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress); @@ -430,12 +429,35 @@ public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws Ignite assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException); } + /** + * Tests that {@link SegmentAware#onSegmentCompressed} returns segments in proper order. + */ + public void testLastCompressedIdxProperOrdering() throws IgniteInterruptedCheckedException { + SegmentAware aware = new SegmentAware(10, true); + + for (int i = 0; i < 5 ; i++) { + aware.setLastArchivedAbsoluteIndex(i); + aware.waitNextSegmentToCompress(); + } + + aware.onSegmentCompressed(0); + + aware.onSegmentCompressed(4); + assertEquals(0, aware.lastCompressedIdx()); + aware.onSegmentCompressed(1); + assertEquals(1, aware.lastCompressedIdx()); + aware.onSegmentCompressed(3); + assertEquals(1, aware.lastCompressedIdx()); + aware.onSegmentCompressed(2); + assertEquals(4, aware.lastCompressedIdx()); + } + /** * Segment reserve correctly. */ public void testReserveCorrectly() { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); //when: reserve one segment twice and one segment once. aware.reserve(5); @@ -478,7 +500,7 @@ public void testReserveCorrectly() { */ public void testAssertFail_WhenReleaseUnreservedSegment() { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.reserve(5); try { @@ -497,7 +519,7 @@ public void testAssertFail_WhenReleaseUnreservedSegment() { */ public void testReserveWorkSegmentCorrectly() { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); //when: lock one segment twice. aware.checkCanReadArchiveOrReserveWorkSegment(5); @@ -530,7 +552,7 @@ public void testReserveWorkSegmentCorrectly() { */ public void testAssertFail_WhenReleaseUnreservedWorkSegment() { //given: thread which awaited segment. - SegmentAware aware = new SegmentAware(10); + SegmentAware aware = new SegmentAware(10, false); aware.checkCanReadArchiveOrReserveWorkSegment(5); try { diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index a9f2601656aea..7631834f9fc66 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsWithCompactionTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest; import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest; import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest; @@ -156,6 +157,7 @@ public static void addRealPageStoreTests(TestSuite suite) { suite.addTestSuite(IgnitePdsExchangeDuringCheckpointTest.class); suite.addTestSuite(IgnitePdsReserveWalSegmentsTest.class); + suite.addTestSuite(IgnitePdsReserveWalSegmentsWithCompactionTest.class); // new style folders with generated consistent ID test suite.addTestSuite(IgniteUidAsConsistentIdMigrationTest.class); From 58dfe061cf8b4c18ac57fa762a559d711cfbf274 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Tue, 2 Oct 2018 18:26:20 +0300 Subject: [PATCH 002/403] IGNITE-9761 Fixed deadlock in WAL manager - Fixes #4890. Signed-off-by: Alexey Goncharuk --- .../wal/aware/SegmentArchivedStorage.java | 8 ++-- .../persistence/wal/aware/SegmentAware.java | 9 ++++ .../wal/aware/SegmentLockStorage.java | 27 +++++------- .../wal/aware/SegmentObservable.java | 10 ++--- .../wal/aware/SegmentAwareTest.java | 42 ++++++++++++++++++- 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java index 1ed607e9b7dd8..c526ae11e6feb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java @@ -63,10 +63,12 @@ long lastArchivedAbsoluteIndex() { /** * @param lastAbsArchivedIdx New value of last archived segment index. */ - synchronized void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) { - this.lastAbsArchivedIdx = lastAbsArchivedIdx; + void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) { + synchronized (this) { + this.lastAbsArchivedIdx = lastAbsArchivedIdx; - notifyAll(); + notifyAll(); + } notifyObservers(lastAbsArchivedIdx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java index 6ba03991e58a0..e46d93f015947 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java @@ -219,6 +219,15 @@ public boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) { return lastArchivedAbsoluteIndex() >= absIdx || segmentLockStorage.lockWorkSegment(absIdx); } + /** + * Visible for test. + * + * @param absIdx Segment absolute index. segment later, use {@link #releaseWorkSegment} for unlock + */ + void lockWorkSegment(long absIdx) { + segmentLockStorage.lockWorkSegment(absIdx); + } + /** * @param absIdx Segment absolute index. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java index 2e145e7789f43..f638d4d57be70 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java @@ -17,8 +17,8 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.aware; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; /** @@ -29,7 +29,7 @@ public class SegmentLockStorage extends SegmentObservable { * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may come from * {@link FileWriteAheadLogManager.RecordsIterator} during WAL replay. Map itself is guarded by this. */ - private Map locked = new HashMap<>(); + private Map locked = new ConcurrentHashMap<>(); /** * Check if WAL segment locked (protected from move to archive) @@ -37,7 +37,7 @@ public class SegmentLockStorage extends SegmentObservable { * @param absIdx Index for check reservation. * @return {@code True} if index is locked. */ - public synchronized boolean locked(long absIdx) { + public boolean locked(long absIdx) { return locked.containsKey(absIdx); } @@ -47,12 +47,8 @@ public synchronized boolean locked(long absIdx) { * segment later, use {@link #releaseWorkSegment} for unlock */ @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext") - synchronized boolean lockWorkSegment(long absIdx) { - Integer cur = locked.get(absIdx); - - cur = cur == null ? 1 : cur + 1; - - locked.put(absIdx, cur); + boolean lockWorkSegment(long absIdx) { + locked.compute(absIdx, (idx, count) -> count == null ? 1 : count + 1); return false; } @@ -61,15 +57,12 @@ synchronized boolean lockWorkSegment(long absIdx) { * @param absIdx Segment absolute index. */ @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext") - synchronized void releaseWorkSegment(long absIdx) { - Integer cur = locked.get(absIdx); - - assert cur != null && cur >= 1 : "cur=" + cur + ", absIdx=" + absIdx; + void releaseWorkSegment(long absIdx) { + locked.compute(absIdx, (idx, count) -> { + assert count != null && count >= 1 : "cur=" + count + ", absIdx=" + absIdx; - if (cur == 1) - locked.remove(absIdx); - else - locked.put(absIdx, cur - 1); + return count == 1 ? null : count - 1; + }); notifyObservers(absIdx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java index ba5ad300cd127..3e915044dd01b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java @@ -17,8 +17,8 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.aware; -import java.util.ArrayList; -import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Consumer; /** @@ -26,12 +26,12 @@ */ public abstract class SegmentObservable { /** Observers for handle changes of archived index. */ - private final List> observers = new ArrayList<>(); + private final Queue> observers = new ConcurrentLinkedQueue<>(); /** * @param observer Observer for notification about segment's changes. */ - synchronized void addObserver(Consumer observer) { + void addObserver(Consumer observer) { observers.add(observer); } @@ -40,7 +40,7 @@ synchronized void addObserver(Consumer observer) { * * @param segmentId Segment which was been changed. */ - synchronized void notifyObservers(long segmentId) { + void notifyObservers(long segmentId) { observers.forEach(observer -> observer.accept(segmentId)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java index 7840b0979e06c..08693564768c6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java @@ -31,6 +31,46 @@ * Test for {@link SegmentAware}. */ public class SegmentAwareTest extends TestCase { + + /** + * Checking to avoid deadlock SegmentArchivedStorage.markAsMovedToArchive -> SegmentLockStorage.locked <-> + * SegmentLockStorage.releaseWorkSegment -> SegmentArchivedStorage.onSegmentUnlocked + * + * @throws IgniteCheckedException if failed. + */ + public void testAvoidDeadlockArchiverAndLockStorage() throws IgniteCheckedException { + SegmentAware aware = new SegmentAware(10, false); + + int iterationCnt = 100_000; + int segmentToHandle = 1; + + IgniteInternalFuture archiverThread = GridTestUtils.runAsync(() -> { + int i = iterationCnt; + + while (i-- > 0) { + try { + aware.markAsMovedToArchive(segmentToHandle); + } + catch (IgniteInterruptedCheckedException e) { + throw new RuntimeException(e); + } + } + }); + + IgniteInternalFuture lockerThread = GridTestUtils.runAsync(() -> { + int i = iterationCnt; + + while (i-- > 0) { + aware.lockWorkSegment(segmentToHandle); + + aware.releaseWorkSegment(segmentToHandle); + } + }); + + archiverThread.get(); + lockerThread.get(); + } + /** * Waiting finished when work segment is set. */ @@ -435,7 +475,7 @@ public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws Ignite public void testLastCompressedIdxProperOrdering() throws IgniteInterruptedCheckedException { SegmentAware aware = new SegmentAware(10, true); - for (int i = 0; i < 5 ; i++) { + for (int i = 0; i < 5; i++) { aware.setLastArchivedAbsoluteIndex(i); aware.waitNextSegmentToCompress(); } From 0bae33501e12ed27322fcba65736676ba06bb4b8 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Tue, 2 Oct 2018 19:07:36 +0300 Subject: [PATCH 003/403] IGNITE-9760 Fixed NPE in WAL manager for FSYNC mode - Fixes #4888. Signed-off-by: Alexey Goncharuk --- .../FsyncModeFileWriteAheadLogManager.java | 7 ++-- .../WalRolloverRecordLoggingFsyncTest.java | 32 +++++++++++++++++++ .../WalRolloverRecordLoggingLogOnlyTest.java | 32 +++++++++++++++++++ .../db/wal/WalRolloverRecordLoggingTest.java | 16 ++++++---- .../IgnitePdsWithIndexingCoreTestSuite.java | 6 ++-- 5 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index 1c0325e46cd8a..3d1b0e0304d2f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -2330,19 +2330,22 @@ public static long writeSerializerVersion(FileIO io, long idx, int version, WALM private abstract static class FileHandle { /** I/O interface for read/write operations with file */ protected SegmentIO fileIO; + /** Segment idx corresponded to fileIo*/ + final long segmentIdx; /** * @param fileIO I/O interface for read/write operations of FileHandle. */ - private FileHandle(SegmentIO fileIO) { + private FileHandle(@NotNull SegmentIO fileIO) { this.fileIO = fileIO; + this.segmentIdx = fileIO.getSegmentId(); } /** * @return Current segment id. */ public long getSegmentId(){ - return fileIO.getSegmentId(); + return segmentIdx; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java new file mode 100644 index 0000000000000..7454e5f68dfe2 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal; + +import org.apache.ignite.configuration.WALMode; +import org.jetbrains.annotations.NotNull; + +/** + * + */ +public class WalRolloverRecordLoggingFsyncTest extends WalRolloverRecordLoggingTest { + + /** {@inheritDoc} */ + @NotNull @Override public WALMode walMode() { + return WALMode.FSYNC; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java new file mode 100644 index 0000000000000..765fdeb5c4556 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal; + +import org.apache.ignite.configuration.WALMode; +import org.jetbrains.annotations.NotNull; + +/** + * + */ +public class WalRolloverRecordLoggingLogOnlyTest extends WalRolloverRecordLoggingTest { + + /** {@inheritDoc} */ + @NotNull @Override public WALMode walMode() { + return WALMode.LOG_ONLY; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java index 67caf63506641..395b03ae4367c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java @@ -23,6 +23,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; import org.apache.ignite.failure.StopNodeOrHaltFailureHandler; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; @@ -36,13 +37,11 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH; -import static org.apache.ignite.configuration.WALMode.LOG_ONLY; +import org.jetbrains.annotations.NotNull; /** * */ -public class WalRolloverRecordLoggingTest extends GridCommonAbstractTest { +public abstract class WalRolloverRecordLoggingTest extends GridCommonAbstractTest { /** */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -69,15 +68,20 @@ private RolloverRecord() { .setDefaultDataRegionConfiguration(new DataRegionConfiguration() .setPersistenceEnabled(true) .setMaxSize(40 * 1024 * 1024)) - .setWalMode(LOG_ONLY) + .setWalMode(walMode()) .setWalSegmentSize(4 * 1024 * 1024) - .setWalArchivePath(DFLT_WAL_PATH)); + ); cfg.setFailureHandler(new StopNodeOrHaltFailureHandler(false, 0)); return cfg; } + /** + * @return Wal mode. + */ + @NotNull public abstract WALMode walMode(); + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { stopAllGrids(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java index caea388359d11..e9159bf7cd553 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java @@ -40,7 +40,8 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryWithCompactionTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalPathsTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest; -import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingFsyncTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingLogOnlyTest; /** * Test suite for tests that cover core PDS features and depend on indexing module. @@ -60,7 +61,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(PersistenceDirectoryWarningLoggingTest.class); suite.addTestSuite(WalPathsTest.class); suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class); - suite.addTestSuite(WalRolloverRecordLoggingTest.class); + suite.addTestSuite(WalRolloverRecordLoggingFsyncTest.class); + suite.addTestSuite(WalRolloverRecordLoggingLogOnlyTest.class); suite.addTestSuite(IgniteWalRecoveryTest.class); suite.addTestSuite(IgniteWalRecoveryWithCompactionTest.class); From 60a8c8c8d0c983ebd62df48d59fdf4b0b40d8336 Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Tue, 2 Oct 2018 19:13:30 +0300 Subject: [PATCH 004/403] GG-14266 Make IGNITE-9084 available from 2.5.3+ --- .../dht/preloader/GridDhtPartitionSupplyMessageV2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java index b6bff0e823506..88240ecc28772 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java @@ -36,7 +36,7 @@ public class GridDhtPartitionSupplyMessageV2 extends GridDhtPartitionSupplyMessa private static final long serialVersionUID = 0L; /** Available since. */ - public static final IgniteProductVersion AVAILABLE_SINCE = IgniteProductVersion.fromString("2.7.0"); + public static final IgniteProductVersion AVAILABLE_SINCE = IgniteProductVersion.fromString("2.5.3"); /** Supplying process error. */ @GridDirectTransient From ad6416ca4905d0d0bd032c636dca55eee497fbdb Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 4 Oct 2018 12:46:54 +0300 Subject: [PATCH 005/403] IGNITE-9661 Improved performance of partition state validation during PME - Fixes #4850. --- .../GridDhtPartitionsStateValidator.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java index d131d56375e4a..544d4531524d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java @@ -119,7 +119,9 @@ public void validatePartitionCountersAndSizes( Set ignore = null; - for (int p = 0; p < top.partitions(); p++) { + for (int i = 0; i < countersMap.size(); i++) { + int p = countersMap.partitionAt(i); + if (top.partitionState(nodeId, p) != GridDhtPartitionState.OWNING) { if (ignore == null) ignore = new HashSet<>(); @@ -129,9 +131,8 @@ public void validatePartitionCountersAndSizes( continue; } - int partIdx = countersMap.partitionIndex(p); - long updateCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0; - long size = sizesMap.containsKey(p) ? sizesMap.get(p) : 0; + long updateCounter = countersMap.updateCounterAt(i); + long size = sizesMap.getOrDefault(p, 0L); // Do not validate partitions with zero update counter and size. if (updateCounter == 0 && size == 0) { @@ -186,14 +187,15 @@ public Map> validatePartitionsUpdateCounters( Set ignorePartitions = shouldIgnore(top, nodeId, e.getValue()); - for (int part = 0; part < partitions; part++) { - if (ignorePartitions != null && ignorePartitions.contains(part)) + for (int i = 0; i < countersMap.size(); i++) { + int p = countersMap.partitionAt(i); + + if (ignorePartitions != null && ignorePartitions.contains(p)) continue; - int partIdx = countersMap.partitionIndex(part); - long currentCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0; + long currentCounter = countersMap.updateCounterAt(i); - process(invalidPartitions, updateCountersAndNodesByPartitions, part, nodeId, currentCounter); + process(invalidPartitions, updateCountersAndNodesByPartitions, p, nodeId, currentCounter); } } @@ -237,17 +239,20 @@ public Map> validatePartitionsSizes( if (ignoringNodes.contains(nodeId)) continue; + CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions); Map sizesMap = e.getValue().partitionSizes(top.groupId()); Set ignorePartitions = shouldIgnore(top, nodeId, e.getValue()); - for (int part = 0; part < partitions; part++) { - if (ignorePartitions != null && ignorePartitions.contains(part)) + for (int i = 0; i < countersMap.size(); i++) { + int p = countersMap.partitionAt(i); + + if (ignorePartitions != null && ignorePartitions.contains(p)) continue; - long currentSize = sizesMap.containsKey(part) ? sizesMap.get(part) : 0L; + long currentSize = sizesMap.getOrDefault(p, 0L); - process(invalidPartitions, sizesAndNodesByPartitions, part, nodeId, currentSize); + process(invalidPartitions, sizesAndNodesByPartitions, p, nodeId, currentSize); } } From ebea1bc90e6619816826f0bdc4be8f7f98d9eefc Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 5 Oct 2018 16:35:05 +0700 Subject: [PATCH 006/403] IGNITE-9792 Fixed assert in case if IGNITE_MBEANS_DISABLED is true. Fixed tests. (cherry picked from commit 78c2d3bbbd620bb7795d9f362785e073d2dec0a2) --- .../org/apache/ignite/internal/IgniteKernal.java | 4 +++- .../junits/IgniteTestResources.java | 14 +++++++++++--- .../junits/common/GridCommonAbstractTest.java | 4 ++-- .../util/mbeans/GridMBeanDisableSelfTest.java | 16 +++++++++------- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 6b1c9956717af..32e5dd8fb983f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1352,7 +1352,9 @@ private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedExcep private void validateCommon(IgniteConfiguration cfg) { A.notNull(cfg.getNodeId(), "cfg.getNodeId()"); - A.notNull(cfg.getMBeanServer(), "cfg.getMBeanServer()"); + if (!U.IGNITE_MBEANS_DISABLED) + A.notNull(cfg.getMBeanServer(), "cfg.getMBeanServer()"); + A.notNull(cfg.getGridLogger(), "cfg.getGridLogger()"); A.notNull(cfg.getMarshaller(), "cfg.getMarshaller()"); A.notNull(cfg.getUserAttributes(), "cfg.getUserAttributes()"); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java index 5fef8bc52c3ee..59e29941331d7 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java @@ -78,6 +78,13 @@ public class IgniteTestResources { /** */ private GridResourceProcessor rsrcProc; + /** + * @return Default MBean server or {@code null} if {@code IGNITE_MBEANS_DISABLED} is configured. + */ + @Nullable private static MBeanServer prepareMBeanServer() { + return U.IGNITE_MBEANS_DISABLED ? null : ManagementFactory.getPlatformMBeanServer(); + } + /** * @throws IgniteCheckedException If failed. */ @@ -87,7 +94,8 @@ public IgniteTestResources() throws IgniteCheckedException { else log = rootLog.getLogger(getClass()); - this.jmx = ManagementFactory.getPlatformMBeanServer(); + this.jmx = prepareMBeanServer(); + this.rsrcProc = new GridResourceProcessor(new GridTestKernalContext(this.log)); } @@ -97,7 +105,7 @@ public IgniteTestResources() throws IgniteCheckedException { public IgniteTestResources(IgniteConfiguration cfg) throws IgniteCheckedException { this.cfg = cfg; this.log = rootLog.getLogger(getClass()); - this.jmx = ManagementFactory.getPlatformMBeanServer(); + this.jmx = prepareMBeanServer(); this.rsrcProc = new GridResourceProcessor(new GridTestKernalContext(this.log, this.cfg)); } @@ -119,7 +127,7 @@ public IgniteTestResources(IgniteLogger log) throws IgniteCheckedException { assert log != null; this.log = log.getLogger(getClass()); - this.jmx = ManagementFactory.getPlatformMBeanServer(); + this.jmx = prepareMBeanServer(); this.rsrcProc = new GridResourceProcessor(new GridTestKernalContext(this.log)); } diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index f3e2b27ceab28..7e98ec763e3aa 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -481,7 +481,7 @@ protected GridNearCacheAdapter near(int idx, String cache) { } /** {@inheritDoc} */ - @Override protected final void setUp() throws Exception { + @Override protected void setUp() throws Exception { // Disable SSL hostname verifier. HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { @Override public boolean verify(String s, SSLSession sslSes) { @@ -495,7 +495,7 @@ protected GridNearCacheAdapter near(int idx, String cache) { } /** {@inheritDoc} */ - @Override protected final void tearDown() throws Exception { + @Override protected void tearDown() throws Exception { getTestCounters().incrementStopped(); super.tearDown(); diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java index ac8f011748c24..b98cd46c369fe 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java @@ -33,15 +33,17 @@ */ public class GridMBeanDisableSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { + @Override protected void setUp() throws Exception { IgniteUtils.IGNITE_MBEANS_DISABLED = true; - super.beforeTestsStarted(); + super.setUp(); } /** {@inheritDoc} */ - @Override protected void afterTestsStopped() throws Exception { + @Override protected void tearDown() throws Exception { IgniteUtils.IGNITE_MBEANS_DISABLED = false; + + super.tearDown(); } /** @@ -52,12 +54,12 @@ public class GridMBeanDisableSelfTest extends GridCommonAbstractTest { public void testCorrectMBeanInfo() throws Exception { // Node should start and stopped with no errors. try (final Ignite ignite = startGrid(0)) { - final MBeanServer server = ignite.configuration().getMBeanServer(); + final MBeanServer srv = ignite.configuration().getMBeanServer(); GridTestUtils.assertThrowsWithCause( new Callable() { @Override public Void call() throws Exception { - U.registerMBean(server, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class); + U.registerMBean(srv, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class); return null; } @@ -72,7 +74,7 @@ public void testCorrectMBeanInfo() throws Exception { "DummyMbean2" ); - U.registerMBean(server, objName, new DummyMBeanImpl(), DummyMBean.class); + U.registerMBean(srv, objName, new DummyMBeanImpl(), DummyMBean.class); return null; @@ -109,4 +111,4 @@ static class DummyMBeanImpl implements DummyMBean { // No op. } } -} \ No newline at end of file +} From 7c9bd23c345bb4217af5c7ae1a4afb1798072039 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Tue, 9 Oct 2018 14:33:25 +0300 Subject: [PATCH 007/403] IGNITE-9785 Introduce read-only state in local node context - Fixes #4907. Signed-off-by: Ivan Rakov (cherry picked from commit 179b09b) --- .../cache/GridCacheSharedContext.java | 17 +++ .../dht/GridDhtTopologyFutureAdapter.java | 3 + .../datastreamer/DataStreamerImpl.java | 47 +++++- .../ClusterReadOnlyModeAbstractTest.java | 114 +++++++++++++++ .../cache/ClusterReadOnlyModeTest.java | 134 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite5.java | 5 +- .../cache/ClusterReadOnlyModeSqlTest.java | 94 ++++++++++++ .../IgniteCacheWithIndexingTestSuite.java | 3 + 8 files changed, 407 insertions(+), 10 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index e4d398a3990be..caa3d2012f151 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -171,6 +171,9 @@ public class GridCacheSharedContext { /** */ private final List stateAwareMgrs; + /** Cluster is in read-only mode. */ + private volatile boolean readOnlyMode; + /** * @param kernalCtx Context. * @param txMgr Transaction manager. @@ -1108,4 +1111,18 @@ public void finishDhtAtomicUpdate(GridCacheVersion ver) { private int dhtAtomicUpdateIndex(GridCacheVersion ver) { return U.safeAbs(ver.hashCode()) % dhtAtomicUpdCnt.length(); } + + /** + * @return {@code true} if cluster is in read-only mode. + */ + public boolean readOnlyMode() { + return readOnlyMode; + } + + /** + * @param readOnlyMode Read-only flag. + */ + public void readOnlyMode(boolean readOnlyMode) { + this.readOnlyMode = readOnlyMode; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java index 539fef48bda46..92143080c2409 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java @@ -97,6 +97,9 @@ protected final CacheValidation validateCacheGroup(CacheGroupContext grp, Collec cctx.name()); } + if (cctx.shared().readOnlyMode() && !read) + return new IgniteCheckedException("Failed to perform cache operation (cluster is in read only mode)" ); + if (grp.needsRecovery() || grp.topologyValidator() != null) { CacheValidation validation = grpValidRes.get(grp.groupId()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index bf1e13db1d377..e86f65313d21a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -85,10 +85,10 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheFutureImpl; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.dr.GridDrType; @@ -725,7 +725,7 @@ private void loadData(Collection entries, GridFutur keys.add(new KeyCacheObjectWrapper(e.getKey())); } - load0(entries, fut, keys, 0); + load0(entries, fut, keys, 0, null, null); } /** {@inheritDoc} */ @@ -798,13 +798,18 @@ private void acquireRemapSemaphore() throws IgniteInterruptedCheckedException { * @param resFut Result future. * @param activeKeys Active keys. * @param remaps Remaps count. + * @param remapNode Node for remap. In case update with {@code allowOverride() == false} fails on one node, + * we don't need to send update request to all affinity nodes again, if topology version does not changed. + * @param remapTopVer Topology version. */ private void load0( Collection entries, final GridFutureAdapter resFut, @Nullable final Collection activeKeys, - final int remaps - ) { + final int remaps, + ClusterNode remapNode, + AffinityTopologyVersion remapTopVer + ) { try { assert entries != null; @@ -876,7 +881,10 @@ else if (rcvr != null) if (key.partition() == -1) key.partition(cctx.affinity().partition(key, false)); - nodes = nodes(key, topVer, cctx); + if (!allowOverwrite() && remapNode != null && F.eq(topVer, remapTopVer)) + nodes = Collections.singletonList(remapNode); + else + nodes = nodes(key, topVer, cctx); } catch (IgniteCheckedException e) { resFut.onDone(e); @@ -903,6 +911,7 @@ else if (rcvr != null) } for (final Map.Entry> e : mappings.entrySet()) { + final ClusterNode node = e.getKey(); final UUID nodeId = e.getKey().id(); Buffer buf = bufMappings.get(nodeId); @@ -964,7 +973,7 @@ else if (remaps + 1 > maxRemapCnt) { if (cancelled) closedException(); - load0(entriesForNode, resFut, activeKeys, remaps + 1); + load0(entriesForNode, resFut, activeKeys, remaps + 1, node, topVer); } catch (Throwable ex) { resFut.onDone( @@ -2195,12 +2204,27 @@ protected static class IsolatedUpdater implements StreamReceiver cctx = internalCache.context(); + +/* todo: uncomment this and remove topFut choosing logic below after IGNITE-9550 race is fixed + GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture(); + AffinityTopologyVersion topVer = topFut.topologyVersion(); +*/ AffinityTopologyVersion topVer = cctx.isLocal() ? cctx.affinity().affinityTopologyVersion() : cctx.shared().exchange().readyAffinityVersion(); + GridDhtTopologyFuture topFut = (GridDhtTopologyFuture)cctx.shared().exchange().affinityReadyFuture(topVer); + + if (topFut == null) { + // Exchange for newer topology version is already in progress, let's try to use last finished future. + GridDhtTopologyFuture lastFinishedFut = cctx.shared().exchange().lastFinishedFuture(); + + if (F.eq(lastFinishedFut.topologyVersion(), topVer)) + topFut = lastFinishedFut; + } + GridCacheVersion ver = cctx.versions().isolatedStreamerVersion(); long ttl = CU.TTL_ETERNAL; @@ -2260,6 +2284,15 @@ else if (ttl == CU.TTL_NOT_CHANGED) expiryTime = CU.toExpireTime(ttl); } + if (topFut != null) { + topFut.get(); + + Throwable err = topFut.validateCache(cctx, false, false, entry.key(), null); + + if (err != null) + throw new IgniteCheckedException(err); + } + boolean primary = cctx.affinity().primaryByKey(cctx.localNode(), entry.key(), topVer); entry.initialValue(e.getValue(), diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java new file mode 100644 index 0000000000000..7cb9649a4ffb8 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.Collection; +import java.util.Collections; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheMode.REPLICATED; + +/** + * + */ +public class ClusterReadOnlyModeAbstractTest extends GridCommonAbstractTest { + /** */ + private static final int SRVS = 3; + + /** Replicated atomic cache. */ + private static final String REPL_ATOMIC_CACHE = "repl_atomic_cache"; + + /** Replicated transactional cache. */ + private static final String REPL_TX_CACHE = "repl_tx_cache"; + + /** Partitioned atomic cache. */ + private static final String PART_ATOMIC_CACHE = "part_atomic_cache"; + + /** Partitioned transactional cache. */ + private static final String PART_TX_CACHE = "part_tx_cache"; + + /** Cache names. */ + protected static final Collection CACHE_NAMES = F.asList(REPL_ATOMIC_CACHE, REPL_TX_CACHE, + PART_ATOMIC_CACHE, PART_TX_CACHE); + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGridsMultiThreaded(SRVS); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + changeClusterReadOnlyMode(false); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setCacheConfiguration( + cacheConfiguration(REPL_ATOMIC_CACHE, REPLICATED, ATOMIC, null), + cacheConfiguration(REPL_TX_CACHE, REPLICATED, TRANSACTIONAL, null), + cacheConfiguration(PART_ATOMIC_CACHE, PARTITIONED, ATOMIC, "part_grp"), + cacheConfiguration(PART_TX_CACHE, PARTITIONED, TRANSACTIONAL, "part_grp") + ); + + return cfg; + } + + /** + * @param cacheMode Cache mode. + * @param atomicityMode Atomicity mode. + * @param grpName Cache group name. + */ + private CacheConfiguration cacheConfiguration(String name, CacheMode cacheMode, + CacheAtomicityMode atomicityMode, String grpName) { + return new CacheConfiguration() + .setName(name) + .setCacheMode(cacheMode) + .setAtomicityMode(atomicityMode) + .setGroupName(grpName) + .setQueryEntities(Collections.singletonList(new QueryEntity(Integer.class, Integer.class))); + } + + /** + * Change read only mode on all nodes. + * + * @param readOnly Read only. + */ + protected void changeClusterReadOnlyMode(boolean readOnly) { + for (int idx = 0; idx < SRVS; idx++) { + IgniteEx ignite = grid(idx); + + ignite.context().cache().context().readOnlyMode(readOnly); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java new file mode 100644 index 0000000000000..ab57614a9e86d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.Random; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl; +import org.apache.ignite.internal.util.typedef.G; + +/** + * Tests cache get/put/remove and data streaming in read-only cluster mode. + */ +public class ClusterReadOnlyModeTest extends ClusterReadOnlyModeAbstractTest { + /** + * Tests cache get/put/remove. + */ + public void testCacheGetPutRemove() { + assertCachesReadOnlyMode(false); + + changeClusterReadOnlyMode(true); + + assertCachesReadOnlyMode(true); + + changeClusterReadOnlyMode(false); + + assertCachesReadOnlyMode(false); + } + + /** + * Tests data streamer. + */ + public void testDataStreamerReadOnly() { + assertDataStreamerReadOnlyMode(false); + + changeClusterReadOnlyMode(true); + + assertDataStreamerReadOnlyMode(true); + + changeClusterReadOnlyMode(false); + + assertDataStreamerReadOnlyMode(false); + } + + /** + * Asserts that all caches in read-only or in read/write mode on all nodes. + * + * @param readOnly If {@code true} then cache must be in read only mode, else in read/write mode. + */ + private void assertCachesReadOnlyMode(boolean readOnly) { + Random rnd = new Random(); + + for (Ignite ignite : G.allGrids()) { + for (String cacheName : CACHE_NAMES) { + IgniteCache cache = ignite.cache(cacheName); + + for (int i = 0; i < 10; i++) { + cache.get(rnd.nextInt(100)); // All gets must succeed. + + if (readOnly) { + // All puts must fail. + try { + cache.put(rnd.nextInt(100), rnd.nextInt()); + + fail("Put must fail for cache " + cacheName); + } + catch (Exception e) { + // No-op. + } + + // All removes must fail. + try { + cache.remove(rnd.nextInt(100)); + + fail("Remove must fail for cache " + cacheName); + } + catch (Exception e) { + // No-op. + } + } + else { + cache.put(rnd.nextInt(100), rnd.nextInt()); // All puts must succeed. + + cache.remove(rnd.nextInt(100)); // All removes must succeed. + } + } + } + } + } + + /** + * @param readOnly If {@code true} then data streamer must fail, else succeed. + */ + private void assertDataStreamerReadOnlyMode(boolean readOnly) { + Random rnd = new Random(); + + for (Ignite ignite : G.allGrids()) { + for (String cacheName : CACHE_NAMES) { + boolean failed = false; + + try (IgniteDataStreamer streamer = ignite.dataStreamer(cacheName)) { + for (int i = 0; i < 10; i++) { + ((DataStreamerImpl)streamer).maxRemapCount(5); + + streamer.addData(rnd.nextInt(1000), rnd.nextInt()); + } + } + catch (CacheException ignored) { + failed = true; + } + + if (failed != readOnly) + fail("Streaming to " + cacheName + " must " + (readOnly ? "fail" : "succeed")); + } + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index dafc44f306081..a583317824b6f 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest; import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest; import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest; +import org.apache.ignite.internal.processors.cache.ClusterReadOnlyModeTest; import org.apache.ignite.internal.processors.cache.ClusterStatePartitionedSelfTest; import org.apache.ignite.internal.processors.cache.ClusterStateReplicatedSelfTest; import org.apache.ignite.internal.processors.cache.ConcurrentCacheStartTest; @@ -43,10 +44,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.NotMappedPartitionInTxTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheAtomicPreloadSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.IgniteCacheAtomicProtocolTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.IgniteCacheContainsKeyColocatedAtomicSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.IgniteCacheContainsKeyNearAtomicSelfTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest; import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest; @@ -82,6 +80,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(ClusterStatePartitionedSelfTest.class); suite.addTestSuite(ClusterStateReplicatedSelfTest.class); + suite.addTestSuite(ClusterReadOnlyModeTest.class); suite.addTestSuite(IgniteCachePartitionLossPolicySelfTest.class); suite.addTestSuite(IgniteCacheGroupsPartitionLossPolicySelfTest.class); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java new file mode 100644 index 0000000000000..d431f73fb0576 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.Random; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.query.FieldsQueryCursor; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.internal.util.typedef.G; + +/** + * Tests SQL queries in read-only cluster mode. + */ +public class ClusterReadOnlyModeSqlTest extends ClusterReadOnlyModeAbstractTest { + /** + * + */ + public void testSqlReadOnly() { + assertSqlReadOnlyMode(false); + + changeClusterReadOnlyMode(true); + + assertSqlReadOnlyMode(true); + + changeClusterReadOnlyMode(false); + + assertSqlReadOnlyMode(false); + } + + /** + * @param readOnly If {@code true} then data modification SQL queries must fail, else succeed. + */ + private void assertSqlReadOnlyMode(boolean readOnly) { + Random rnd = new Random(); + + for (Ignite ignite : G.allGrids()) { + for (String cacheName : CACHE_NAMES) { + IgniteCache cache = ignite.cache(cacheName); + + try (FieldsQueryCursor cur = cache.query(new SqlFieldsQuery("SELECT * FROM Integer"))) { + cur.getAll(); + } + + boolean failed = false; + + try (FieldsQueryCursor cur = cache.query(new SqlFieldsQuery("DELETE FROM Integer"))) { + cur.getAll(); + } + catch (CacheException ex) { + if (!readOnly) + log.error("Failed to delete data", ex); + + failed = true; + } + + if (failed != readOnly) + fail("SQL delete from " + cacheName + " must " + (readOnly ? "fail" : "succeed")); + + failed = false; + + try (FieldsQueryCursor cur = cache.query(new SqlFieldsQuery( + "INSERT INTO Integer(_KEY, _VAL) VALUES (?, ?)").setArgs(rnd.nextInt(1000), rnd.nextInt()))) { + cur.getAll(); + } + catch (CacheException ex) { + if (!readOnly) + log.error("Failed to insert data", ex); + + failed = true; + } + + if (failed != readOnly) + fail("SQL insert into " + cacheName + " must " + (readOnly ? "fail" : "succeed")); + } + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index e351cb6f4a106..8517ebbf2a503 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.CacheQueryFilterExpiredTest; import org.apache.ignite.internal.processors.cache.CacheRandomOperationsMultithreadedTest; import org.apache.ignite.internal.processors.cache.ClientReconnectAfterClusterRestartTest; +import org.apache.ignite.internal.processors.cache.ClusterReadOnlyModeSqlTest; import org.apache.ignite.internal.processors.cache.GridCacheOffHeapSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheOffheapIndexEntryEvictTest; import org.apache.ignite.internal.processors.cache.GridCacheOffheapIndexGetSelfTest; @@ -90,6 +91,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(BinaryTypeMismatchLoggingTest.class); + suite.addTestSuite(ClusterReadOnlyModeSqlTest.class); + return suite; } } From 25e94c30b1de41ac6d3e459a5a466a0cb803a0a0 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Wed, 10 Oct 2018 11:57:11 +0300 Subject: [PATCH 008/403] IGNITE-9500: SQL: implemented system view for list of caches (IGNITE.CACHES). This closes #4716. --- .../processors/query/h2/IgniteH2Indexing.java | 2 + .../sys/view/SqlAbstractLocalSystemView.java | 13 +- .../h2/sys/view/SqlSystemViewCaches.java | 197 +++++++++++++++ .../h2/sys/view/SqlSystemViewNodeMetrics.java | 2 +- .../query/h2/sys/view/SqlSystemViewNodes.java | 2 +- .../query/SqlSystemViewsSelfTest.java | 225 ++++++++++++++++++ 6 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index fe563b71e9034..7c5f2746631e3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -131,6 +131,7 @@ import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine; import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemView; import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewBaselineNodes; +import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewCaches; import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeAttributes; import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeMetrics; import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes; @@ -3268,6 +3269,7 @@ public Collection systemViews(GridKernalContext ctx) { views.add(new SqlSystemViewNodeAttributes(ctx)); views.add(new SqlSystemViewBaselineNodes(ctx)); views.add(new SqlSystemViewNodeMetrics(ctx)); + views.add(new SqlSystemViewCaches(ctx)); return views; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java index d692dbac3dafa..d028406a122c9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java @@ -37,7 +37,7 @@ public abstract class SqlAbstractLocalSystemView extends SqlAbstractSystemView { * @param tblName Table name. * @param desc Description. * @param ctx Context. - * @param indexes Indexed columns. + * @param indexes Indexes. * @param cols Columns. */ public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext ctx, String[] indexes, @@ -49,6 +49,17 @@ public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext assert indexes != null; } + /** + * @param tblName Table name. + * @param desc Description. + * @param ctx Context. + * @param indexedCols Indexed columns. + * @param cols Columns. + */ + public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext ctx, String indexedCols, Column... cols) { + this(tblName, desc, ctx, new String[] {indexedCols}, cols); + } + /** * @param tblName Table name. * @param desc Description. diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java new file mode 100644 index 0000000000000..ff9ef32115121 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.sys.view; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; +import org.apache.ignite.internal.util.typedef.F; +import org.h2.engine.Session; +import org.h2.result.Row; +import org.h2.result.SearchRow; +import org.h2.value.Value; + +/** + * System view: caches. + */ +public class SqlSystemViewCaches extends SqlAbstractLocalSystemView { + /** + * @param ctx Grid context. + */ + public SqlSystemViewCaches(GridKernalContext ctx) { + super("CACHES", "Ignite caches", ctx, "NAME", + newColumn("NAME"), + newColumn("CACHE_ID", Value.INT), + newColumn("CACHE_TYPE"), + newColumn("GROUP_ID", Value.INT), + newColumn("GROUP_NAME"), + newColumn("CACHE_MODE"), + newColumn("ATOMICITY_MODE"), + newColumn("IS_ONHEAP_CACHE_ENABLED", Value.BOOLEAN), + newColumn("IS_COPY_ON_READ", Value.BOOLEAN), + newColumn("IS_LOAD_PREVIOUS_VALUE", Value.BOOLEAN), + newColumn("IS_READ_FROM_BACKUP", Value.BOOLEAN), + newColumn("PARTITION_LOSS_POLICY"), + newColumn("NODE_FILTER"), + newColumn("TOPOLOGY_VALIDATOR"), + newColumn("IS_EAGER_TTL", Value.BOOLEAN), + newColumn("WRITE_SYNCHRONIZATION_MODE"), + newColumn("IS_INVALIDATE", Value.BOOLEAN), + newColumn("IS_EVENTS_DISABLED", Value.BOOLEAN), + newColumn("IS_STATISTICS_ENABLED", Value.BOOLEAN), + newColumn("IS_MANAGEMENT_ENABLED", Value.BOOLEAN), + newColumn("BACKUPS", Value.INT), + newColumn("AFFINITY"), + newColumn("AFFINITY_MAPPER"), + newColumn("REBALANCE_MODE"), + newColumn("REBALANCE_BATCH_SIZE", Value.INT), + newColumn("REBALANCE_TIMEOUT", Value.LONG), + newColumn("REBALANCE_DELAY", Value.LONG), + newColumn("REBALANCE_THROTTLE", Value.LONG), + newColumn("REBALANCE_BATCHES_PREFETCH_COUNT", Value.LONG), + newColumn("REBALANCE_ORDER", Value.INT), + newColumn("EVICTION_FILTER"), + newColumn("EVICTION_POLICY_FACTORY"), + newColumn("IS_NEAR_CACHE_ENABLED", Value.BOOLEAN), + newColumn("NEAR_CACHE_EVICTION_POLICY_FACTORY"), + newColumn("NEAR_CACHE_START_SIZE", Value.INT), + newColumn("DEFAULT_LOCK_TIMEOUT", Value.LONG), + newColumn("CACHE_INTERCEPTOR"), + newColumn("CACHE_STORE_FACTORY"), + newColumn("IS_STORE_KEEP_BINARY", Value.BOOLEAN), + newColumn("IS_READ_THROUGH", Value.BOOLEAN), + newColumn("IS_WRITE_THROUGH", Value.BOOLEAN), + newColumn("IS_WRITE_BEHIND_ENABLED", Value.BOOLEAN), + newColumn("WRITE_BEHIND_COALESCING", Value.BOOLEAN), + newColumn("WRITE_BEHIND_FLUSH_SIZE", Value.INT), + newColumn("WRITE_BEHIND_FLUSH_FREQUENCY", Value.LONG), + newColumn("WRITE_BEHIND_FLUSH_THREAD_COUNT", Value.INT), + newColumn("WRITE_BEHIND_FLUSH_BATCH_SIZE", Value.INT), + newColumn("MAX_CONCURRENT_ASYNC_OPERATIONS", Value.INT), + newColumn("CACHE_LOADER_FACTORY"), + newColumn("CACHE_WRITER_FACTORY"), + newColumn("EXPIRY_POLICY_FACTORY"), + newColumn("IS_SQL_ESCAPE_ALL", Value.BOOLEAN), + newColumn("SQL_SCHEMA"), + newColumn("SQL_INDEX_MAX_INLINE_SIZE", Value.INT), + newColumn("IS_SQL_ONHEAP_CACHE_ENABLED", Value.BOOLEAN), + newColumn("SQL_ONHEAP_CACHE_MAX_SIZE", Value.INT), + newColumn("QUERY_DETAILS_METRICS_SIZE", Value.INT), + newColumn("QUERY_PARALLELISM", Value.INT), + newColumn("MAX_QUERY_ITERATORS_COUNT", Value.INT), + newColumn("DATA_REGION_NAME") + ); + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public Iterator getRows(Session ses, SearchRow first, SearchRow last) { + SqlSystemViewColumnCondition nameCond = conditionForColumn("NAME", first, last); + + Collection caches; + + if (nameCond.isEquality()) { + DynamicCacheDescriptor cache = ctx.cache().cacheDescriptor(nameCond.valueForEquality().getString()); + + caches = cache == null ? Collections.emptySet() : Collections.singleton(cache); + } + else + caches = ctx.cache().cacheDescriptors().values(); + + AtomicLong rowKey = new AtomicLong(); + + return F.iterator(caches, + cache -> createRow(ses, rowKey.incrementAndGet(), + cache.cacheName(), + cache.cacheId(), + cache.cacheType(), + cache.groupId(), + cache.groupDescriptor().groupName(), + cache.cacheConfiguration().getCacheMode(), + cache.cacheConfiguration().getAtomicityMode(), + cache.cacheConfiguration().isOnheapCacheEnabled(), + cache.cacheConfiguration().isCopyOnRead(), + cache.cacheConfiguration().isLoadPreviousValue(), + cache.cacheConfiguration().isReadFromBackup(), + cache.cacheConfiguration().getPartitionLossPolicy(), + cache.cacheConfiguration().getNodeFilter(), + cache.cacheConfiguration().getTopologyValidator(), + cache.cacheConfiguration().isEagerTtl(), + cache.cacheConfiguration().getWriteSynchronizationMode(), + cache.cacheConfiguration().isInvalidate(), + cache.cacheConfiguration().isEventsDisabled(), + cache.cacheConfiguration().isStatisticsEnabled(), + cache.cacheConfiguration().isManagementEnabled(), + cache.cacheConfiguration().getBackups(), + cache.cacheConfiguration().getAffinity(), + cache.cacheConfiguration().getAffinityMapper(), + cache.cacheConfiguration().getRebalanceMode(), + cache.cacheConfiguration().getRebalanceBatchSize(), + cache.cacheConfiguration().getRebalanceTimeout(), + cache.cacheConfiguration().getRebalanceDelay(), + cache.cacheConfiguration().getRebalanceThrottle(), + cache.cacheConfiguration().getRebalanceBatchesPrefetchCount(), + cache.cacheConfiguration().getRebalanceOrder(), + cache.cacheConfiguration().getEvictionFilter(), + cache.cacheConfiguration().getEvictionPolicyFactory(), + cache.cacheConfiguration().getNearConfiguration() != null, + cache.cacheConfiguration().getNearConfiguration() != null ? + cache.cacheConfiguration().getNearConfiguration().getNearEvictionPolicyFactory() : null, + cache.cacheConfiguration().getNearConfiguration() != null ? + cache.cacheConfiguration().getNearConfiguration().getNearStartSize() : null, + cache.cacheConfiguration().getDefaultLockTimeout(), + cache.cacheConfiguration().getInterceptor(), + cache.cacheConfiguration().getCacheStoreFactory(), + cache.cacheConfiguration().isStoreKeepBinary(), + cache.cacheConfiguration().isReadThrough(), + cache.cacheConfiguration().isWriteThrough(), + cache.cacheConfiguration().isWriteBehindEnabled(), + cache.cacheConfiguration().getWriteBehindCoalescing(), + cache.cacheConfiguration().getWriteBehindFlushSize(), + cache.cacheConfiguration().getWriteBehindFlushFrequency(), + cache.cacheConfiguration().getWriteBehindFlushThreadCount(), + cache.cacheConfiguration().getWriteBehindBatchSize(), + cache.cacheConfiguration().getMaxConcurrentAsyncOperations(), + cache.cacheConfiguration().getCacheLoaderFactory(), + cache.cacheConfiguration().getCacheWriterFactory(), + cache.cacheConfiguration().getExpiryPolicyFactory(), + cache.cacheConfiguration().isSqlEscapeAll(), + cache.cacheConfiguration().getSqlSchema(), + cache.cacheConfiguration().getSqlIndexMaxInlineSize(), + cache.cacheConfiguration().isSqlOnheapCacheEnabled(), + cache.cacheConfiguration().getSqlOnheapCacheMaxSize(), + cache.cacheConfiguration().getQueryDetailMetricsSize(), + cache.cacheConfiguration().getQueryParallelism(), + cache.cacheConfiguration().getMaxQueryIteratorsCount(), + cache.cacheConfiguration().getDataRegionName() + ), true); + } + + /** {@inheritDoc} */ + @Override public boolean canGetRowCount() { + return true; + } + + /** {@inheritDoc} */ + @Override public long getRowCount() { + return ctx.cache().cacheDescriptors().size(); + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java index 01b4e976f0cae..d3921aaaa0490 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java @@ -40,7 +40,7 @@ public class SqlSystemViewNodeMetrics extends SqlAbstractLocalSystemView { * @param ctx Grid context. */ public SqlSystemViewNodeMetrics(GridKernalContext ctx) { - super("NODE_METRICS", "Node metrics", ctx, new String[] {"NODE_ID"}, + super("NODE_METRICS", "Node metrics", ctx, "NODE_ID", newColumn("NODE_ID", Value.UUID), newColumn("LAST_UPDATE_TIME", Value.TIMESTAMP), newColumn("MAX_ACTIVE_JOBS", Value.INT), diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java index 514f92e9708ee..d8720310aef08 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java @@ -39,7 +39,7 @@ public class SqlSystemViewNodes extends SqlAbstractLocalSystemView { * @param ctx Grid context. */ public SqlSystemViewNodes(GridKernalContext ctx) { - super("NODES", "Topology nodes", ctx, new String[] {"ID"}, + super("NODES", "Topology nodes", ctx, "ID", newColumn("ID", Value.UUID), newColumn("CONSISTENT_ID"), newColumn("VERSION"), diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java index ccd07964fba29..feee9a0bedd1f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java @@ -19,24 +19,35 @@ import java.sql.Time; import java.sql.Timestamp; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Random; import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.Callable; +import javax.cache.Cache; +import javax.cache.configuration.Factory; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.eviction.EvictableEntry; +import org.apache.ignite.cache.eviction.EvictionFilter; +import org.apache.ignite.cache.eviction.EvictionPolicy; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterMetrics; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.TopologyValidator; import org.apache.ignite.internal.ClusterMetricsSnapshot; import org.apache.ignite.internal.IgniteNodeAttributes; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; +import org.apache.ignite.internal.util.lang.GridNodePredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.X; @@ -504,6 +515,154 @@ public void testBaselineViews() throws Exception { return super.getConfiguration().setCacheConfiguration(new CacheConfiguration().setName(DEFAULT_CACHE_NAME)); } + /** + * Test caches system views. + */ + public void testCachesViews() throws Exception { + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setName("def").setPersistenceEnabled(true)) + .setDataRegionConfigurations(new DataRegionConfiguration().setName("dr1"), + new DataRegionConfiguration().setName("dr2")); + + Ignite ignite0 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg)); + + Ignite ignite1 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg).setIgniteInstanceName("node1")); + + ignite0.cluster().active(true); + + Ignite ignite2 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg).setIgniteInstanceName("node2")); + + Ignite ignite3 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg).setIgniteInstanceName("node3") + .setClientMode(true)); + + ignite0.getOrCreateCache(new CacheConfiguration<>() + .setName("cache_atomic_part") + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setCacheMode(CacheMode.PARTITIONED) + .setGroupName("part_grp") + .setNodeFilter(new TestNodeFilter(ignite0.cluster().localNode())) + ); + + ignite0.getOrCreateCache(new CacheConfiguration<>() + .setName("cache_atomic_repl") + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setCacheMode(CacheMode.REPLICATED) + .setDataRegionName("dr1") + .setTopologyValidator(new TestTopologyValidator()) + ); + + ignite0.getOrCreateCache(new CacheConfiguration<>() + .setName("cache_tx_part") + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setCacheMode(CacheMode.PARTITIONED) + .setGroupName("part_grp") + .setNodeFilter(new TestNodeFilter(ignite0.cluster().localNode())) + ); + + ignite0.getOrCreateCache(new CacheConfiguration<>() + .setName("cache_tx_repl") + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setCacheMode(CacheMode.REPLICATED) + .setDataRegionName("dr2") + .setEvictionFilter(new TestEvictionFilter()) + .setEvictionPolicyFactory(new TestEvictionPolicyFactory()) + .setOnheapCacheEnabled(true) + ); + + execSql("CREATE TABLE cache_sql (ID INT PRIMARY KEY, VAL VARCHAR) WITH " + + "\"cache_name=cache_sql,template=partitioned,atomicity=atomic\""); + + awaitPartitionMapExchange(); + + List> resAll = execSql("SELECT NAME, CACHE_ID, CACHE_TYPE, GROUP_ID, GROUP_NAME, " + + "CACHE_MODE, ATOMICITY_MODE, IS_ONHEAP_CACHE_ENABLED, IS_COPY_ON_READ, IS_LOAD_PREVIOUS_VALUE, " + + "IS_READ_FROM_BACKUP, PARTITION_LOSS_POLICY, NODE_FILTER, TOPOLOGY_VALIDATOR, IS_EAGER_TTL, " + + "WRITE_SYNCHRONIZATION_MODE, IS_INVALIDATE, IS_EVENTS_DISABLED, IS_STATISTICS_ENABLED, " + + "IS_MANAGEMENT_ENABLED, BACKUPS, AFFINITY, AFFINITY_MAPPER, " + + "REBALANCE_MODE, REBALANCE_BATCH_SIZE, REBALANCE_TIMEOUT, REBALANCE_DELAY, REBALANCE_THROTTLE, " + + "REBALANCE_BATCHES_PREFETCH_COUNT, REBALANCE_ORDER, " + + "EVICTION_FILTER, EVICTION_POLICY_FACTORY, " + + "IS_NEAR_CACHE_ENABLED, NEAR_CACHE_EVICTION_POLICY_FACTORY, NEAR_CACHE_START_SIZE, " + + "DEFAULT_LOCK_TIMEOUT, CACHE_INTERCEPTOR, CACHE_STORE_FACTORY, " + + "IS_STORE_KEEP_BINARY, IS_READ_THROUGH, IS_WRITE_THROUGH, " + + "IS_WRITE_BEHIND_ENABLED, WRITE_BEHIND_COALESCING, WRITE_BEHIND_FLUSH_SIZE, " + + "WRITE_BEHIND_FLUSH_FREQUENCY, WRITE_BEHIND_FLUSH_THREAD_COUNT, WRITE_BEHIND_FLUSH_BATCH_SIZE, " + + "MAX_CONCURRENT_ASYNC_OPERATIONS, CACHE_LOADER_FACTORY, CACHE_WRITER_FACTORY, EXPIRY_POLICY_FACTORY, " + + "IS_SQL_ESCAPE_ALL, SQL_SCHEMA, SQL_INDEX_MAX_INLINE_SIZE, IS_SQL_ONHEAP_CACHE_ENABLED, " + + "SQL_ONHEAP_CACHE_MAX_SIZE, QUERY_DETAILS_METRICS_SIZE, QUERY_PARALLELISM, MAX_QUERY_ITERATORS_COUNT, " + + "DATA_REGION_NAME FROM IGNITE.CACHES"); + + assertColumnTypes(resAll.get(0), + String.class, Integer.class, String.class, Integer.class, String.class, + String.class, String.class, Boolean.class, Boolean.class, Boolean.class, + Boolean.class, String.class, String.class, String.class, Boolean.class, + String.class, Boolean.class, Boolean.class, Boolean.class, + Boolean.class, Integer.class, String.class, String.class, + String.class, Integer.class, Long.class, Long.class, Long.class, // Rebalance. + Long.class, Integer.class, + String.class, String.class, // Eviction. + Boolean.class, String.class, Integer.class, // Near cache. + Long.class, String.class, String.class, + Boolean.class, Boolean.class, Boolean.class, + Boolean.class, Boolean.class, Integer.class, // Write-behind. + Long.class, Integer.class, Integer.class, + Integer.class, String.class, String.class, String.class, + Boolean.class, String.class, Integer.class, Boolean.class, // SQL. + Integer.class, Integer.class, Integer.class, Integer.class, + String.class); + + assertEquals("cache_tx_part", execSql("SELECT NAME FROM IGNITE.CACHES WHERE " + + "CACHE_MODE = 'PARTITIONED' AND ATOMICITY_MODE = 'TRANSACTIONAL' AND NAME like 'cache%'").get(0).get(0)); + + assertEquals("cache_atomic_repl", execSql("SELECT NAME FROM IGNITE.CACHES WHERE " + + "CACHE_MODE = 'REPLICATED' AND ATOMICITY_MODE = 'ATOMIC' AND NAME like 'cache%'").get(0).get(0)); + + assertEquals(2L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE GROUP_NAME = 'part_grp'") + .get(0).get(0)); + + assertEquals("cache_atomic_repl", execSql("SELECT NAME FROM IGNITE.CACHES " + + "WHERE DATA_REGION_NAME = 'dr1'").get(0).get(0)); + + assertEquals("cache_tx_repl", execSql("SELECT NAME FROM IGNITE.CACHES " + + "WHERE DATA_REGION_NAME = 'dr2'").get(0).get(0)); + + assertEquals("PARTITIONED", execSql("SELECT CACHE_MODE FROM IGNITE.CACHES " + + "WHERE NAME = 'cache_atomic_part'").get(0).get(0)); + + assertEquals("USER", execSql("SELECT CACHE_TYPE FROM IGNITE.CACHES WHERE NAME = 'cache_sql'") + .get(0).get(0)); + + assertEquals(0L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME = 'no_such_cache'").get(0) + .get(0)); + + assertEquals(0L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME = 1").get(0).get(0)); + + assertEquals("TestNodeFilter", execSql("SELECT NODE_FILTER FROM IGNITE.CACHES WHERE NAME = " + + "'cache_atomic_part'").get(0).get(0)); + + assertEquals("TestEvictionFilter", execSql("SELECT EVICTION_FILTER FROM IGNITE.CACHES " + + "WHERE NAME = 'cache_tx_repl'").get(0).get(0)); + + assertEquals("TestEvictionPolicyFactory", execSql("SELECT EVICTION_POLICY_FACTORY " + + "FROM IGNITE.CACHES WHERE NAME = 'cache_tx_repl'").get(0).get(0)); + + assertEquals("TestTopologyValidator", execSql("SELECT TOPOLOGY_VALIDATOR FROM IGNITE.CACHES " + + "WHERE NAME = 'cache_atomic_repl'").get(0).get(0)); + + // Check that caches are the same on BLT, BLT filtered by node filter, non BLT and client nodes. + assertEquals(5L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'").get(0) + .get(0)); + + assertEquals(5L, execSql(ignite1, "SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'") + .get(0).get(0)); + + assertEquals(5L, execSql(ignite2, "SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'") + .get(0).get(0)); + + assertEquals(5L, execSql(ignite3, "SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'") + .get(0).get(0)); + } + /** * Gets ignite configuration with persistence enabled. */ @@ -533,4 +692,70 @@ private long convertToMilliseconds(Object sqlTime) { return time0.getTime() + TimeZone.getDefault().getOffset(time0.getTime()); } + + /** + * + */ + private static class TestNodeFilter extends GridNodePredicate { + /** + * @param node Node. + */ + public TestNodeFilter(ClusterNode node) { + super(node); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "TestNodeFilter"; + } + } + + /** + * + */ + private static class TestEvictionFilter implements EvictionFilter { + /** {@inheritDoc} */ + @Override public boolean evictAllowed(Cache.Entry entry) { + return false; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "TestEvictionFilter"; + } + } + + /** + * + */ + private static class TestEvictionPolicyFactory implements Factory> { + /** {@inheritDoc} */ + @Override public EvictionPolicy create() { + return new EvictionPolicy() { + @Override public void onEntryAccessed(boolean rmv, EvictableEntry entry) { + // No-op. + } + }; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "TestEvictionPolicyFactory"; + } + } + + /** + * + */ + private static class TestTopologyValidator implements TopologyValidator { + /** {@inheritDoc} */ + @Override public boolean validate(Collection nodes) { + return true; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "TestTopologyValidator"; + } + } } From 2a03b23c5840923377549d86693cdb74fac25e6a Mon Sep 17 00:00:00 2001 From: Denis Mekhanikov Date: Fri, 5 Oct 2018 16:13:45 +0300 Subject: [PATCH 009/403] IGNITE-9794 Handle UnregisteredBinaryTypeException on metadata registration under topology lock. - Fixes #4916. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit b1206121e7a87f2d84414ab03b86b8614c0bc3c0) --- .../internal/binary/BinaryFieldAccessor.java | 3 +- ...aRegistrationInsideEntryProcessorTest.java | 49 +++++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java index 87c4f3e18d8fc..7d138a30e6d82 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.UUID; import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.internal.UnregisteredBinaryTypeException; import org.apache.ignite.internal.UnregisteredClassException; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.F; @@ -156,7 +157,7 @@ public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectExce write0(obj, writer); } catch (Exception ex) { - if (ex instanceof UnregisteredClassException) + if (ex instanceof UnregisteredClassException || ex instanceof UnregisteredBinaryTypeException) throw ex; if (S.INCLUDE_SENSITIVE && !F.isEmpty(name)) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java index 73dae4bb39a5d..c7c3757b9a701 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java @@ -17,7 +17,7 @@ package org.apache.ignite.internal.processors.cache; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import javax.cache.processor.EntryProcessor; @@ -40,7 +40,7 @@ public class BinaryMetadataRegistrationInsideEntryProcessorTest extends GridComm /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration() { TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder() - .setAddresses(Arrays.asList("127.0.0.1:47500..47509")); + .setAddresses(Collections.singletonList("127.0.0.1:47500..47509")); return new IgniteConfiguration() .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(ipFinder)) @@ -60,9 +60,9 @@ public void test() throws Exception { cache.invoke(i, new CustomProcessor()); } catch (Exception e) { - Map value = cache.get(1); + Map val = cache.get(1); - if ((value != null) && (value.get(1) != null) && (value.get(1).getObj() == CustomEnum.ONE)) + if ((val != null) && (val.get(1).anEnum == CustomEnum.ONE) && val.get(1).obj.data.equals("test")) System.out.println("Data was saved."); else System.out.println("Data wasn't saved."); @@ -82,7 +82,7 @@ private static class CustomProcessor implements EntryProcessor map = new HashMap<>(); - map.put(1, new CustomObj(CustomEnum.ONE)); + map.put(1, new CustomObj(new CustomInnerObject("test"), CustomEnum.ONE)); entry.setValue(map); @@ -95,27 +95,20 @@ private static class CustomProcessor implements EntryProcessor Date: Thu, 11 Oct 2018 10:03:34 +0700 Subject: [PATCH 010/403] IGNITE-7926 Web Agent: Support launching with Java 8+. (cherry picked from commit f313d650448207942357a88bcdeab5833c8bd963) --- .../web-agent/bin/ignite-web-agent.bat | 63 +++++++++++-- .../web-agent/bin/ignite-web-agent.sh | 81 +++++++++-------- .../web-agent/bin/include/functions.sh | 88 +++++++++++++++++++ 3 files changed, 183 insertions(+), 49 deletions(-) create mode 100644 modules/web-console/web-agent/bin/include/functions.sh diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.bat b/modules/web-console/web-agent/bin/ignite-web-agent.bat index 387447f18d616..1c305ca9c02f2 100644 --- a/modules/web-console/web-agent/bin/ignite-web-agent.bat +++ b/modules/web-console/web-agent/bin/ignite-web-agent.bat @@ -43,7 +43,7 @@ goto checkIgniteHome2 if defined JAVA_HOME goto checkJdk echo %0, ERROR: echo JAVA_HOME environment variable is not found. - echo Please point JAVA_HOME variable to location of JDK 1.7 or JDK 1.8. + echo Please point JAVA_HOME variable to location of JDK 1.8 or JDK 9. echo You can also download latest JDK at http://java.com/download. goto error_finish @@ -52,18 +52,31 @@ goto error_finish if exist "%JAVA_HOME%\bin\java.exe" goto checkJdkVersion echo %0, ERROR: echo JAVA is not found in JAVA_HOME=%JAVA_HOME%. - echo Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8. + echo Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9. echo You can also download latest JDK at http://java.com/download. goto error_finish :checkJdkVersion -"%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr "1\.[78]\." > nul -if %ERRORLEVEL% equ 0 goto run_java +set cmd="%JAVA_HOME%\bin\java.exe" +for /f "tokens=* USEBACKQ" %%f in (`%cmd% -version 2^>^&1`) do ( + set var=%%f + goto :LoopEscape +) +:LoopEscape + +for /f "tokens=1-3 delims= " %%a in ("%var%") do set JAVA_VER_STR=%%c +set JAVA_VER_STR=%JAVA_VER_STR:"=% + +for /f "tokens=1,2 delims=." %%a in ("%JAVA_VER_STR%.x") do set MAJOR_JAVA_VER=%%a & set MINOR_JAVA_VER=%%b +if %MAJOR_JAVA_VER% == 1 set MAJOR_JAVA_VER=%MINOR_JAVA_VER% + +if %MAJOR_JAVA_VER% LSS 8 ( echo %0, ERROR: echo The version of JAVA installed in %JAVA_HOME% is incorrect. - echo Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8. + echo Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9. echo You can also download latest JDK at http://java.com/download. -goto error_finish + goto error_finish +) :run_java @@ -74,13 +87,47 @@ goto error_finish :: "%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr "1\.[7]\." > nul if %ERRORLEVEL% equ 0 ( - if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxPermSize=256m + if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:MaxPermSize=256m ) else ( - if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m + if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms1g -Xmx1g -server -XX:MaxMetaspaceSize=256m ) set JVM_OPTS=%JVM_OPTS% -Djava.net.useSystemProxies=true +:: +:: Final JVM_OPTS for Java 9+ compatibility +:: +if "%MAJOR_JAVA_VER%" == "8" ( + set JVM_OPTS= ^ + -XX:+AggressiveOpts ^ + %JVM_OPTS% +) + +if %MAJOR_JAVA_VER% GEQ 9 if %MAJOR_JAVA_VER% LSS 11 ( + set JVM_OPTS= ^ + -XX:+AggressiveOpts ^ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED ^ + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED ^ + --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED ^ + --add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED ^ + --add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED ^ + --illegal-access=permit ^ + --add-modules=java.transaction ^ + --add-modules=java.xml.bind ^ + %JVM_OPTS% +) + +if "%MAJOR_JAVA_VER%" == "11" ( + set JVM_OPTS= ^ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED ^ + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED ^ + --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED ^ + --add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED ^ + --add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED ^ + --illegal-access=permit ^ + %JVM_OPTS% +) + "%JAVA_HOME%\bin\java.exe" %JVM_OPTS% -cp "*" org.apache.ignite.console.agent.AgentLauncher %* set JAVA_ERRORLEVEL=%ERRORLEVEL% diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.sh b/modules/web-console/web-agent/bin/ignite-web-agent.sh index 530bc7f4d6efa..3a649e3aaae09 100755 --- a/modules/web-console/web-agent/bin/ignite-web-agent.sh +++ b/modules/web-console/web-agent/bin/ignite-web-agent.sh @@ -16,49 +16,14 @@ # limitations under the License. # -# Check JAVA_HOME. -if [ "$JAVA_HOME" = "" ]; then - JAVA=`type -p java` - RETCODE=$? - - if [ $RETCODE -ne 0 ]; then - echo $0", ERROR:" - echo "JAVA_HOME environment variable is not found." - echo "Please point JAVA_HOME variable to location of JDK 1.7 or JDK 1.8." - echo "You can also download latest JDK at http://java.com/download" - - exit 1 - fi +SOURCE=$(dirname "$0") - JAVA_HOME= -else - JAVA=${JAVA_HOME}/bin/java -fi +source "${SOURCE}"/include/functions.sh # -# Check JDK. +# Discover path to Java executable and check it's version. # -if [ ! -e "$JAVA" ]; then - echo $0", ERROR:" - echo "JAVA is not found in JAVA_HOME=$JAVA_HOME." - echo "Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8." - echo "You can also download latest JDK at http://java.com/download" - - exit 1 -fi - -JAVA_VER=`"$JAVA" -version 2>&1 | egrep "1\.[78]\."` - -if [ "$JAVA_VER" == "" ]; then - echo $0", ERROR:" - echo "The version of JAVA installed in JAVA_HOME=$JAVA_HOME is incorrect." - echo "Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8." - echo "You can also download latest JDK at http://java.com/download" - - exit 1 -fi - -SOURCE="${BASH_SOURCE[0]}" +checkJava # # Set IGNITE_HOME. @@ -87,12 +52,46 @@ cd $DIR # if [ -z "$JVM_OPTS" ] ; then if [[ `"$JAVA" -version 2>&1 | egrep "1\.[7]\."` ]]; then - JVM_OPTS="-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxPermSize=256m" + JVM_OPTS="-Xms1g -Xmx1g -server -XX:MaxPermSize=256m" else - JVM_OPTS="-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m" + JVM_OPTS="-Xms1g -Xmx1g -server -XX:MaxMetaspaceSize=256m" fi fi JVM_OPTS="${JVM_OPTS} -Djava.net.useSystemProxies=true" +# +# Final JVM_OPTS for Java 9+ compatibility +# +javaMajorVersion "${JAVA_HOME}/bin/java" + +if [ $version -eq 8 ] ; then + JVM_OPTS="\ + -XX:+AggressiveOpts \ + ${JVM_OPTS}" + +elif [ $version -gt 8 ] && [ $version -lt 11 ]; then + JVM_OPTS="\ + -XX:+AggressiveOpts \ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \ + --add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \ + --add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \ + --illegal-access=permit \ + --add-modules=java.transaction \ + --add-modules=java.xml.bind \ + ${JVM_OPTS}" + +elif [ $version -eq 11 ] ; then + JVM_OPTS="\ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \ + --add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \ + --add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \ + --illegal-access=permit \ + ${JVM_OPTS}" +fi + "$JAVA" ${JVM_OPTS} -cp "*" org.apache.ignite.console.agent.AgentLauncher "$@" diff --git a/modules/web-console/web-agent/bin/include/functions.sh b/modules/web-console/web-agent/bin/include/functions.sh new file mode 100644 index 0000000000000..0c316e386a5b2 --- /dev/null +++ b/modules/web-console/web-agent/bin/include/functions.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This is a collection of utility functions to be used in other Ignite scripts. +# Before calling any function from this file you have to import it: +# if [ "${IGNITE_HOME}" = "" ]; +# then IGNITE_HOME_TMP="$(dirname "$(cd "$(dirname "$0")"; "pwd")")"; +# else IGNITE_HOME_TMP=${IGNITE_HOME}; +# fi +# +# source "${IGNITE_HOME_TMP}"/bin/include/functions.sh +# + +# Extract java version to `version` variable. +javaVersion() { + version=$("$1" -version 2>&1 | awk -F '"' '/version/ {print $2}') +} + +# Extract only major version of java to `version` variable. +javaMajorVersion() { + javaVersion "$1" + version="${version%%.*}" + + if [ ${version} -eq 1 ]; then + # Version seems starts from 1, we need second number. + javaVersion "$1" + backIFS=$IFS + + IFS=. ver=(${version##*-}) + version=${ver[1]} + + IFS=$backIFS + fi +} + +# +# Discovers path to Java executable and checks it's version. +# The function exports JAVA variable with path to Java executable. +# +checkJava() { + # Check JAVA_HOME. + if [ "$JAVA_HOME" = "" ]; then + JAVA=`type -p java` + RETCODE=$? + + if [ $RETCODE -ne 0 ]; then + echo $0", ERROR:" + echo "JAVA_HOME environment variable is not found." + echo "Please point JAVA_HOME variable to location of JDK 1.8 or JDK 9." + echo "You can also download latest JDK at http://java.com/download" + + exit 1 + fi + + JAVA_HOME= + else + JAVA=${JAVA_HOME}/bin/java + fi + + # + # Check JDK. + # + javaMajorVersion "$JAVA" + + if [ $version -lt 8 ]; then + echo "$0, ERROR:" + echo "The $version version of JAVA installed in JAVA_HOME=$JAVA_HOME is incompatible." + echo "Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9." + echo "You can also download latest JDK at http://java.com/download" + exit 1 + fi +} From 8a48441e209f88bb428795a6459ad02689a0e04d Mon Sep 17 00:00:00 2001 From: Eduard Shangareev Date: Thu, 11 Oct 2018 10:58:15 +0300 Subject: [PATCH 011/403] IGNITE-9796 NPE if you call array() method on empty GridLongList - Fixes #4917. Signed-off-by: Ivan Rakov (cherry picked from commit 447ce47) --- .../processors/cluster/BaselineTopology.java | 14 ++--- .../ignite/internal/util/GridLongList.java | 6 +++ .../junits/GridAbstractTest.java | 53 +++++++++---------- .../ignite/util/GridLongListSelfTest.java | 23 ++++++++ 4 files changed, 63 insertions(+), 33 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java index fcb7a397e8936..5a7e66aefc7bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java @@ -298,13 +298,15 @@ public ClusterNode baselineNode(Object consId) { * @return Sorted list of baseline topology nodes. */ public List createBaselineView( - List aliveNodes, - @Nullable IgnitePredicate nodeFilter) - { + Collection aliveNodes, + @Nullable IgnitePredicate nodeFilter + ) { List res = new ArrayList<>(nodeMap.size()); + boolean nullNodeFilter = nodeFilter == null; + for (ClusterNode node : aliveNodes) { - if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter))) + if (nodeMap.containsKey(node.consistentId()) && (nullNodeFilter || CU.affinityNode(node, nodeFilter))) res.add(node); } @@ -316,7 +318,7 @@ public List createBaselineView( Map consIdMap = new HashMap<>(); for (ClusterNode node : aliveNodes) { - if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter))) + if (nodeMap.containsKey(node.consistentId()) && (nullNodeFilter || CU.affinityNode(node, nodeFilter))) consIdMap.put(node.consistentId(), node); } @@ -326,7 +328,7 @@ public List createBaselineView( if (!consIdMap.containsKey(consId)) { DetachedClusterNode node = new DetachedClusterNode(consId, e.getValue()); - if (nodeFilter == null || CU.affinityNode(node, nodeFilter)) + if (nullNodeFilter || CU.affinityNode(node, nodeFilter)) consIdMap.put(consId, node); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java index d1f20e634a03c..8d09539590f43 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java @@ -42,6 +42,9 @@ public class GridLongList implements Message, Externalizable { /** */ private static final long serialVersionUID = 0L; + /** Empty array. */ + public static final long[] EMPTY_ARRAY = new long[0]; + /** */ private long[] arr; @@ -390,6 +393,9 @@ public int replaceValue(int startIdx, long oldVal, long newVal) { * @return Array copy. */ public long[] array() { + if (arr == null) + return EMPTY_ARRAY; + long[] res = new long[idx]; System.arraycopy(arr, 0, res, 0, idx); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index 13387de72e6fe..f1d66825b715b 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -17,6 +17,31 @@ package org.apache.ignite.testframework.junits; +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Proxy; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.cache.configuration.Factory; +import javax.cache.configuration.FactoryBuilder; import junit.framework.TestCase; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -98,32 +123,6 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.support.FileSystemXmlApplicationContext; -import javax.cache.configuration.Factory; -import javax.cache.configuration.FactoryBuilder; -import java.io.ObjectStreamException; -import java.io.Serializable; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Proxy; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLIENT_CACHE_CHANGE_MESSAGE_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -604,7 +603,7 @@ protected void afterTestsStopped() throws Exception { info(">>> Starting test class: " + testClassDescription() + " <<<"); if(isSafeTopology()) - assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution"; + assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution:" + G.allGrids(); if (startGrid) { IgniteConfiguration cfg = optimize(getConfiguration()); diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java index 3b62e32f01f06..8849a3d2727d0 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java @@ -150,4 +150,27 @@ public void testSort() { assertEquals(asList(1, 3, 4, 5, 0), list); assertEquals(asList(0, 1, 3, 4, 5), list.sort()); } + + /** + * + */ + public void testArray() { + GridLongList list = new GridLongList(); + + long[] array = list.array(); + + assertNotNull(array); + + assertEquals(0, array.length); + + list.add(1L); + + array = list.array(); + + assertNotNull(array); + + assertEquals(1, array.length); + + assertEquals(1L, array[0]); + } } \ No newline at end of file From 66c309424f3846f18c3254d1916e9b63a866fc7a Mon Sep 17 00:00:00 2001 From: EdShangGG Date: Thu, 11 Oct 2018 22:03:15 +0300 Subject: [PATCH 012/403] fix compilation --- .../wal/aware/SegmentAwareTest.java | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java index b50c9eb5a08b1..22b30f140e552 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java @@ -31,45 +31,6 @@ * Test for {@link SegmentAware}. */ public class SegmentAwareTest extends TestCase { - /** - * Checking to avoid deadlock SegmentArchivedStorage.markAsMovedToArchive -> SegmentLockStorage.locked <-> - * SegmentLockStorage.releaseWorkSegment -> SegmentArchivedStorage.onSegmentUnlocked - * - * @throws IgniteCheckedException if failed. - */ - public void testAvoidDeadlockArchiverAndLockStorage() throws IgniteCheckedException { - SegmentAware aware = new SegmentAware(10); - - int iterationCnt = 100_000; - int segmentToHandle = 1; - - IgniteInternalFuture archiverThread = GridTestUtils.runAsync(() -> { - int i = iterationCnt; - - while (i-- > 0) { - try { - aware.markAsMovedToArchive(segmentToHandle); - } - catch (IgniteInterruptedCheckedException e) { - throw new RuntimeException(e); - } - } - }); - - IgniteInternalFuture lockerThread = GridTestUtils.runAsync(() -> { - int i = iterationCnt; - - while (i-- > 0) { - aware.lockWorkSegment(segmentToHandle); - - aware.releaseWorkSegment(segmentToHandle); - } - }); - - archiverThread.get(); - lockerThread.get(); - } - /** * Checking to avoid deadlock SegmentArchivedStorage.markAsMovedToArchive -> SegmentLockStorage.locked <-> * SegmentLockStorage.releaseWorkSegment -> SegmentArchivedStorage.onSegmentUnlocked From deda24e8424594288ee0c94c16813b200c365df8 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 12 Oct 2018 14:06:06 +0700 Subject: [PATCH 013/403] IGNITE-9863 Refactored VisorDataTransferObject to IgniteDataTransferObject. Deprecated VisorDataTransferObject class. Minor code cleanup. (cherry picked from commit 789046ba49d61e6b58c89555b8d9b419f00052a1) --- .../dto/IgniteDataTransferObject.java | 130 +++++++++++++++ .../dto/IgniteDataTransferObjectInput.java | 156 ++++++++++++++++++ .../dto/IgniteDataTransferObjectOutput.java | 141 ++++++++++++++++ .../visor/VisorDataTransferObject.java | 5 +- .../visor/VisorDataTransferObjectInput.java | 2 + .../visor/VisorDataTransferObjectOutput.java | 2 + .../visor/debug/VisorThreadMonitorInfo.java | 5 - .../visor/event/VisorGridDeploymentEvent.java | 5 - .../visor/event/VisorGridDiscoveryEvent.java | 5 - .../visor/event/VisorGridJobEvent.java | 5 - .../visor/event/VisorGridTaskEvent.java | 5 - 11 files changed, 435 insertions(+), 26 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java new file mode 100644 index 0000000000000..3441742dfabd6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.dto; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import org.jetbrains.annotations.Nullable; + +/** + * Base class for data transfer objects. + */ +public abstract class IgniteDataTransferObject implements Externalizable { + /** */ + private static final long serialVersionUID = 0L; + + /** Magic number to detect correct transfer objects. */ + private static final int MAGIC = 0x42BEEF00; + + /** Version 1. */ + protected static final byte V1 = 1; + + /** Version 2. */ + protected static final byte V2 = 2; + + /** Version 3. */ + protected static final byte V3 = 3; + + /** Version 4. */ + protected static final byte V4 = 4; + + /** Version 5. */ + protected static final byte V5 = 5; + + /** + * @param col Source collection. + * @param Collection type. + * @return List based on passed collection. + */ + @Nullable protected static List toList(Collection col) { + if (col != null) + return new ArrayList<>(col); + + return null; + } + + /** + * @param col Source collection. + * @param Collection type. + * @return List based on passed collection. + */ + @Nullable protected static Set toSet(Collection col) { + if (col != null) + return new LinkedHashSet<>(col); + + return null; + } + + /** + * @return Transfer object version. + */ + public byte getProtocolVersion() { + return V1; + } + + /** + * Save object's specific data content. + * + * @param out Output object to write data content. + * @throws IOException If I/O errors occur. + */ + protected abstract void writeExternalData(ObjectOutput out) throws IOException; + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + int hdr = MAGIC + getProtocolVersion(); + + out.writeInt(hdr); + + try (IgniteDataTransferObjectOutput dtout = new IgniteDataTransferObjectOutput(out)) { + writeExternalData(dtout); + } + } + + /** + * Load object's specific data content. + * + * @param protoVer Input object version. + * @param in Input object to load data content. + * @throws IOException If I/O errors occur. + * @throws ClassNotFoundException If the class for an object being restored cannot be found. + */ + protected abstract void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException; + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + int hdr = in.readInt(); + + if ((hdr & MAGIC) != MAGIC) + throw new IOException("Unexpected IgniteDataTransferObject header " + + "[actual=" + Integer.toHexString(hdr) + ", expected=" + Integer.toHexString(MAGIC) + "]"); + + byte ver = (byte)(hdr & 0xFF); + + try (IgniteDataTransferObjectInput dtin = new IgniteDataTransferObjectInput(in)) { + readExternalData(ver, dtin); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java new file mode 100644 index 0000000000000..c12287520656a --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.dto; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectInputStream; +import org.apache.ignite.internal.util.io.GridByteArrayInputStream; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.NotNull; + +/** + * Wrapper for object input. + */ +public class IgniteDataTransferObjectInput implements ObjectInput { + /** */ + private final ObjectInputStream ois; + + /** + * @param in Target input. + * @throws IOException If an I/O error occurs. + */ + public IgniteDataTransferObjectInput(ObjectInput in) throws IOException { + byte[] buf = U.readByteArray(in); + + /* */ + GridByteArrayInputStream bis = new GridByteArrayInputStream(buf); + ois = new ObjectInputStream(bis); + } + + + /** {@inheritDoc} */ + @Override public Object readObject() throws ClassNotFoundException, IOException { + return ois.readObject(); + } + + /** {@inheritDoc} */ + @Override public int read() throws IOException { + return ois.read(); + } + + /** {@inheritDoc} */ + @Override public int read(byte[] b) throws IOException { + return ois.read(b); + } + + /** {@inheritDoc} */ + @Override public int read(byte[] b, int off, int len) throws IOException { + return ois.read(b, off, len); + } + + /** {@inheritDoc} */ + @Override public long skip(long n) throws IOException { + return ois.skip(n); + } + + /** {@inheritDoc} */ + @Override public int available() throws IOException { + return ois.available(); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + ois.close(); + } + + /** {@inheritDoc} */ + @Override public void readFully(@NotNull byte[] b) throws IOException { + ois.readFully(b); + } + + /** {@inheritDoc} */ + @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException { + ois.readFully(b, off, len); + } + + /** {@inheritDoc} */ + @Override public int skipBytes(int n) throws IOException { + return ois.skipBytes(n); + } + + /** {@inheritDoc} */ + @Override public boolean readBoolean() throws IOException { + return ois.readBoolean(); + } + + /** {@inheritDoc} */ + @Override public byte readByte() throws IOException { + return ois.readByte(); + } + + /** {@inheritDoc} */ + @Override public int readUnsignedByte() throws IOException { + return ois.readUnsignedByte(); + } + + /** {@inheritDoc} */ + @Override public short readShort() throws IOException { + return ois.readShort(); + } + + /** {@inheritDoc} */ + @Override public int readUnsignedShort() throws IOException { + return ois.readUnsignedShort(); + } + + /** {@inheritDoc} */ + @Override public char readChar() throws IOException { + return ois.readChar(); + } + + /** {@inheritDoc} */ + @Override public int readInt() throws IOException { + return ois.readInt(); + } + + /** {@inheritDoc} */ + @Override public long readLong() throws IOException { + return ois.readLong(); + } + + /** {@inheritDoc} */ + @Override public float readFloat() throws IOException { + return ois.readFloat(); + } + + /** {@inheritDoc} */ + @Override public double readDouble() throws IOException { + return ois.readDouble(); + } + + /** {@inheritDoc} */ + @Override public String readLine() throws IOException { + return ois.readLine(); + } + + /** {@inheritDoc} */ + @NotNull @Override public String readUTF() throws IOException { + return ois.readUTF(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java new file mode 100644 index 0000000000000..db4933cea29c0 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.dto; + +import java.io.IOException; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import org.apache.ignite.internal.util.io.GridByteArrayOutputStream; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.NotNull; + +/** + * Wrapper for object output. + */ +public class IgniteDataTransferObjectOutput implements ObjectOutput { + /** */ + private final ObjectOutput out; + + /** */ + private final GridByteArrayOutputStream bos; + + /** */ + private final ObjectOutputStream oos; + + /** + * Constructor. + * + * @param out Target stream. + * @throws IOException If an I/O error occurs. + */ + public IgniteDataTransferObjectOutput(ObjectOutput out) throws IOException { + this.out = out; + + bos = new GridByteArrayOutputStream(); + oos = new ObjectOutputStream(bos); + } + + /** {@inheritDoc} */ + @Override public void writeObject(Object obj) throws IOException { + oos.writeObject(obj); + } + + /** {@inheritDoc} */ + @Override public void write(int b) throws IOException { + oos.write(b); + } + + /** {@inheritDoc} */ + @Override public void write(byte[] b) throws IOException { + oos.write(b); + } + + /** {@inheritDoc} */ + @Override public void write(byte[] b, int off, int len) throws IOException { + oos.write(b, off, len); + } + + /** {@inheritDoc} */ + @Override public void writeBoolean(boolean v) throws IOException { + oos.writeBoolean(v); + } + + /** {@inheritDoc} */ + @Override public void writeByte(int v) throws IOException { + oos.writeByte(v); + } + + /** {@inheritDoc} */ + @Override public void writeShort(int v) throws IOException { + oos.writeShort(v); + } + + /** {@inheritDoc} */ + @Override public void writeChar(int v) throws IOException { + oos.writeChar(v); + } + + /** {@inheritDoc} */ + @Override public void writeInt(int v) throws IOException { + oos.writeInt(v); + } + + /** {@inheritDoc} */ + @Override public void writeLong(long v) throws IOException { + oos.writeLong(v); + } + + /** {@inheritDoc} */ + @Override public void writeFloat(float v) throws IOException { + oos.writeFloat(v); + } + + /** {@inheritDoc} */ + @Override public void writeDouble(double v) throws IOException { + oos.writeDouble(v); + } + + /** {@inheritDoc} */ + @Override public void writeBytes(@NotNull String s) throws IOException { + oos.writeBytes(s); + } + + /** {@inheritDoc} */ + @Override public void writeChars(@NotNull String s) throws IOException { + oos.writeChars(s); + } + + /** {@inheritDoc} */ + @Override public void writeUTF(@NotNull String s) throws IOException { + oos.writeUTF(s); + } + + /** {@inheritDoc} */ + @Override public void flush() throws IOException { + oos.flush(); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + oos.flush(); + + U.writeByteArray(out, bos.internalArray(), bos.size()); + + oos.close(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java index 38d7a0ad0aef5..6ba3aa7d5aed5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java @@ -26,10 +26,13 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; import org.jetbrains.annotations.Nullable; /** - * Base class for data transfer objects. + * Base class for data transfer objects for Visor tasks. + * + * @deprecated Use {@link IgniteDataTransferObject} instead. This class may be removed in Ignite 3.0. */ public abstract class VisorDataTransferObject implements Externalizable { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java index 16e933090eb9b..7caf26914ba5e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java @@ -20,12 +20,14 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectInputStream; +import org.apache.ignite.internal.dto.IgniteDataTransferObjectInput; import org.apache.ignite.internal.util.io.GridByteArrayInputStream; import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.NotNull; /** * Wrapper for object input. + * @deprecated Use {@link IgniteDataTransferObjectInput} instead. This class may be removed in Ignite 3.0. */ public class VisorDataTransferObjectInput implements ObjectInput { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java index 7fa772e75ca13..4dbd25035bac9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java @@ -20,12 +20,14 @@ import java.io.IOException; import java.io.ObjectOutput; import java.io.ObjectOutputStream; +import org.apache.ignite.internal.dto.IgniteDataTransferObjectOutput; import org.apache.ignite.internal.util.io.GridByteArrayOutputStream; import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.NotNull; /** * Wrapper for object output. + * @deprecated Use {@link IgniteDataTransferObjectOutput} instead. This class may be removed in Ignite 3.0. */ public class VisorDataTransferObjectOutput implements ObjectOutput { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java index f8ddc68a31e3a..189b8e3cdfba4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java @@ -71,11 +71,6 @@ public StackTraceElement getStackFrame() { return stackFrame; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return 1; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java index 8b0c2110368b9..e74b54c896c34 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java @@ -79,11 +79,6 @@ public String getAlias() { return alias; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return 1; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java index ec2220d1fd7ba..c5f1d30c53d92 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java @@ -119,11 +119,6 @@ public long getTopologyVersion() { return topVer; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return 1; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java index 734d85fa11cb7..2a7ea1862e396 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java @@ -118,11 +118,6 @@ public IgniteUuid getJobId() { return jobId; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return 1; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java index 11c9a17d38dd5..a0836c481356f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java @@ -118,11 +118,6 @@ public boolean isInternal() { return internal; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return 1; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) { From 16a41da9f369a41fb30384eea95bb27ac73a839f Mon Sep 17 00:00:00 2001 From: pkonstantinov Date: Fri, 12 Oct 2018 15:14:04 +0300 Subject: [PATCH 014/403] IGNITE-9606: JDBC: fixed primary key column name resolution in metadata. This closes #4906. This closes #4948. --- .../jdbc/suite/IgniteJdbcDriverTestSuite.java | 2 + .../JdbcThinMetadataPrimaryKeysSelfTest.java | 152 ++++++++++++++++++ .../jdbc/thin/JdbcThinMetadataSelfTest.java | 2 +- .../odbc/jdbc/JdbcRequestHandler.java | 5 +- 4 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataPrimaryKeysSelfTest.java diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java index 2e98d689deaa6..747b13e9d356d 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java @@ -66,6 +66,7 @@ import org.apache.ignite.jdbc.thin.JdbcThinLocalQueriesSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.JdbcThinMetadataPrimaryKeysSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinMetadataSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinMissingLongArrayResultsTest; import org.apache.ignite.jdbc.thin.JdbcThinNoDefaultSchemaTest; @@ -166,6 +167,7 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(JdbcThinSchemaCaseTest.class)); suite.addTest(new TestSuite(JdbcThinEmptyCacheSelfTest.class)); suite.addTest(new TestSuite(JdbcThinMetadataSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinMetadataPrimaryKeysSelfTest.class)); suite.addTest(new TestSuite(JdbcThinErrorsSelfTest.class)); suite.addTest(new TestSuite(JdbcThinInsertStatementSelfTest.class)); diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataPrimaryKeysSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataPrimaryKeysSelfTest.java new file mode 100644 index 0000000000000..5733b72fef2ae --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataPrimaryKeysSelfTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.jdbc.thin; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Verifies that primary keys in the metadata are valid. + */ +public class JdbcThinMetadataPrimaryKeysSelfTest extends GridCommonAbstractTest { + /** Url. */ + private static final String URL = "jdbc:ignite:thin://127.0.0.1"; + + /** COLUMN_NAME column index in the metadata table. */ + private static final int COL_NAME_IDX = 4; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrid(1); + } + + /** + * Execute update sql operation using new connection. + * + * @param sql update SQL query. + * @return update count. + * @throws SQLException on error. + */ + private int executeUpdate(String sql) throws SQLException { + try (Connection conn = DriverManager.getConnection(URL)) { + try (PreparedStatement stmt = conn.prepareStatement(sql)) { + return stmt.executeUpdate(); + } + } + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + executeUpdate("DROP TABLE IF EXISTS TEST;"); + } + + /** + * Checks for PK that contains single unwrapped field. + */ + public void testSingleUnwrappedKey() throws Exception { + executeUpdate("CREATE TABLE TEST (ID LONG PRIMARY KEY, NAME VARCHAR);"); + + checkPKFields("TEST", "ID"); + } + + /** + * Checks for PK that contains single field. Key is forcibly wrapped. + */ + public void testSingleWrappedKey() throws Exception { + executeUpdate("CREATE TABLE TEST (" + + "ID LONG PRIMARY KEY, " + + "NAME VARCHAR) " + + "WITH \"wrap_key=true\";"); + + checkPKFields("TEST", "ID"); + } + + /** + * Checks for composite (so implicitly wrapped) primary key. + */ + public void testCompositeKey() throws Exception { + executeUpdate("CREATE TABLE TEST (" + + "ID LONG, " + + "SEC_ID LONG, " + + "NAME VARCHAR, " + + "PRIMARY KEY (ID, SEC_ID));"); + + checkPKFields("TEST", "ID", "SEC_ID"); + } + + /** + * Checks for composite (so implicitly wrapped) primary key. Additionally, affinity key is used. + */ + public void testCompositeKeyWithAK() throws Exception { + final String tpl = "CREATE TABLE TEST (" + + "ID LONG, " + + "SEC_ID LONG, " + + "NAME VARCHAR, " + + "PRIMARY KEY (ID, SEC_ID)) " + + "WITH \"affinity_key=%s\";"; + + executeUpdate(String.format(tpl, "ID")); + + checkPKFields("TEST", "ID", "SEC_ID"); + + executeUpdate("DROP TABLE TEST;"); + + executeUpdate(String.format(tpl, "SEC_ID")); + + checkPKFields("TEST", "ID", "SEC_ID"); + } + + /** + * Checks that field names in the metadata matches specified expected fields. + * + * @param tabName part of the sql query after CREATE TABLE TESTER. + * @param expPKFields Expected primary key fields. + */ + private void checkPKFields(String tabName, String... expPKFields) throws Exception { + try (Connection conn = DriverManager.getConnection(URL)) { + DatabaseMetaData md = conn.getMetaData(); + + ResultSet rs = md.getPrimaryKeys(conn.getCatalog(), "", tabName); + + List colNames = new ArrayList<>(); + + while (rs.next()) + colNames.add(rs.getString(COL_NAME_IDX)); + + assertEquals("Field names in the primary key are not correct", + Arrays.asList(expPKFields), colNames); + } + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java index 59382f1854704..35832b79d5c8f 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java @@ -555,7 +555,7 @@ public void testGetAllPrimaryKeys() throws Exception { "PUBLIC.TEST.PK_PUBLIC_TEST.ID", "PUBLIC.TEST.PK_PUBLIC_TEST.NAME", "PUBLIC.Quoted.PK_PUBLIC_Quoted.Id", - "PUBLIC.TEST_DECIMAL_COLUMN.ID._KEY")); + "PUBLIC.TEST_DECIMAL_COLUMN.ID.ID")); Set actualPks = new HashSet<>(expectedPks.size()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java index d59788c3d6d9b..d5a277e5a391a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java @@ -1013,8 +1013,11 @@ private ClientListenerResponse getPrimaryKeys(JdbcMetaPrimaryKeysRequest req) { table.keyFieldName(); if (fields.isEmpty()) { + String keyColName = + table.keyFieldName() == null ? QueryUtils.KEY_FIELD_NAME : table.keyFieldName(); + meta.add(new JdbcPrimaryKeyMeta(table.schemaName(), table.tableName(), keyName, - Collections.singletonList("_KEY"))); + Collections.singletonList(keyColName))); } else meta.add(new JdbcPrimaryKeyMeta(table.schemaName(), table.tableName(), keyName, fields)); From e727d8cc3440f780efce2e26dd56ebaf228d5ba0 Mon Sep 17 00:00:00 2001 From: Andrey Kuznetsov Date: Fri, 12 Oct 2018 21:45:53 +0300 Subject: [PATCH 015/403] IGNITE-9601 Write rollover WAL record as the last record in current segment - Fixes #4762. Signed-off-by: Ivan Rakov (cherry picked from commit f07bdaa) --- .../wal/IgniteWriteAheadLogManager.java | 19 +- .../pagemem/wal/record/RolloverType.java | 38 ++ .../pagemem/wal/record/SnapshotRecord.java | 5 - .../pagemem/wal/record/WALRecord.java | 7 - .../wal/FileWriteAheadLogManager.java | 76 +++- .../FsyncModeFileWriteAheadLogManager.java | 51 ++- .../db/wal/WalRolloverRecordLoggingTest.java | 8 +- .../db/wal/WalRolloverTypesTest.java | 366 ++++++++++++++++++ .../persistence/pagemem/NoOpWALManager.java | 9 +- .../StandaloneWalRecordsIteratorTest.java | 3 +- .../testsuites/IgnitePdsTestSuite2.java | 3 + 11 files changed, 536 insertions(+), 49 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/RolloverType.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverTypesTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java index 4ffa347985e59..68428d08a4c91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedManager; import org.apache.ignite.internal.processors.cache.persistence.StorageException; @@ -52,7 +53,7 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni /** * Appends the given log entry to the write-ahead log. * - * @param entry entry to log. + * @param entry Entry to log. * @return WALPointer that may be passed to {@link #flush(WALPointer, boolean)} method to make sure the record is * written to the log. * @throws IgniteCheckedException If failed to construct log entry. @@ -60,6 +61,22 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni */ public WALPointer log(WALRecord entry) throws IgniteCheckedException, StorageException; + /** + * Appends the given log entry to the write-ahead log. If entry logging leads to rollover, caller can specify + * whether to write the entry to the current segment or to th next one. + * + * @param entry Entry to log. + * @param rolloverType Rollover type. + * @return WALPointer that may be passed to {@link #flush(WALPointer, boolean)} method to make sure the record is + * written to the log. + * @throws IgniteCheckedException If failed to construct log entry. + * @throws StorageException If IO error occurred while writing log entry. + * + * @see RolloverType + */ + public WALPointer log(WALRecord entry, RolloverType rolloverType) + throws IgniteCheckedException, StorageException; + /** * Makes sure that all log entries written to the log up until the specified pointer are actually written * to the underlying storage. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/RolloverType.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/RolloverType.java new file mode 100644 index 0000000000000..1d99de1e79741 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/RolloverType.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.pagemem.wal.record; + +/** + * Defines WAL logging type with regard to segment rollover. + */ +public enum RolloverType { + /** Record being logged is not a rollover record. */ + NONE, + + /** + * Record being logged is a rollover record and it should get to the current segment whenever possible. + * If current segment is full, then the record gets to the next segment. Anyway, logging implementation should + * guarantee segment rollover afterwards. + */ + CURRENT_SEGMENT, + + /** + * Record being logged is a rollover record and it should become the first record in the next segment. + */ + NEXT_SEGMENT; +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java index c6b63295c781b..caa1494b00e3e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java @@ -51,11 +51,6 @@ public boolean isFull() { return full; } - /** {@inheritDoc} */ - @Override public boolean rollOver() { - return true; - } - /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java index 667f8d941e249..e22adc09b1815 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java @@ -290,13 +290,6 @@ public void size(int size) { this.size = size; } - /** - * @return Need wal rollOver. - */ - public boolean rollOver(){ - return false; - } - /** * @return Entry type. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index fb604bc247fa5..fe0efbd0e820a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -45,6 +45,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; @@ -82,6 +83,7 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; @@ -95,15 +97,15 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; +import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; -import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory; +import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; -import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl; @@ -706,7 +708,7 @@ private void checkWalConfiguration() throws IgniteCheckedException { "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() + ", oldVer=" + currHnd.serializer.version() + ']'); - rollOver(currHnd); + rollOver(currHnd, null); } currHnd.resume = false; @@ -788,7 +790,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { try { handle.buf.close(); - rollOver(handle); + rollOver(handle, null); } catch (IgniteCheckedException e) { U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e); @@ -798,8 +800,12 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { } /** {@inheritDoc} */ - @SuppressWarnings("TooBroadScope") - @Override public WALPointer log(WALRecord rec) throws IgniteCheckedException, StorageException { + @Override public WALPointer log(WALRecord rec) throws IgniteCheckedException { + return log(rec, RolloverType.NONE); + } + + /** {@inheritDoc} */ + @Override public WALPointer log(WALRecord rec, RolloverType rolloverType) throws IgniteCheckedException { if (serializer == null || mode == WALMode.NONE) return null; @@ -815,21 +821,32 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { rec.size(serializer.size(rec)); while (true) { - if (rec.rollOver()) { - assert cctx.database().checkpointLockIsHeldByThread(); + WALPointer ptr; - long idx = currWrHandle.getSegmentId(); + if (rolloverType == RolloverType.NONE) + ptr = currWrHandle.addRecord(rec); + else { + assert cctx.database().checkpointLockIsHeldByThread(); - currWrHandle.buf.close(); + if (rolloverType == RolloverType.NEXT_SEGMENT) { + WALPointer pos = rec.position(); - currWrHandle = rollOver(currWrHandle); + do { + // This will change rec.position() unless concurrent rollover happened. + currWrHandle = closeBufAndRollover(currWrHandle, rec, rolloverType); + } + while (Objects.equals(pos, rec.position())); - if (log != null && log.isInfoEnabled()) - log.info("Rollover segment [" + idx + " to " + currWrHandle.getSegmentId() + "], recordType=" + rec.type()); + ptr = rec.position(); + } + else if (rolloverType == RolloverType.CURRENT_SEGMENT) { + if ((ptr = currWrHandle.addRecord(rec)) != null) + currWrHandle = closeBufAndRollover(currWrHandle, rec, rolloverType); + } + else + throw new IgniteCheckedException("Unknown rollover type: " + rolloverType); } - WALPointer ptr = currWrHandle.addRecord(rec); - if (ptr != null) { metrics.onWalRecordLogged(); @@ -841,7 +858,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { return ptr; } else - currWrHandle = rollOver(currWrHandle); + currWrHandle = rollOver(currWrHandle, null); checkNode(); @@ -850,6 +867,24 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { } } + /** */ + private FileWriteHandle closeBufAndRollover( + FileWriteHandle currWriteHandle, + WALRecord rec, + RolloverType rolloverType + ) throws IgniteCheckedException { + long idx = currWriteHandle.getSegmentId(); + + currWriteHandle.buf.close(); + + FileWriteHandle res = rollOver(currWriteHandle, rolloverType == RolloverType.NEXT_SEGMENT ? rec : null); + + if (log != null && log.isInfoEnabled()) + log.info("Rollover segment [" + idx + " to " + res.getSegmentId() + "], recordType=" + rec.type()); + + return res; + } + /** {@inheritDoc} */ @Override public void flush(WALPointer ptr, boolean explicitFsync) throws IgniteCheckedException, StorageException { if (serializer == null || mode == WALMode.NONE) @@ -1180,9 +1215,10 @@ private FileWriteHandle currentHandle() { /** * @param cur Handle that failed to fit the given entry. + * @param rec Optional record to be added right after header. * @return Handle that will fit the entry. */ - private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteCheckedException { + private FileWriteHandle rollOver(FileWriteHandle cur, @Nullable WALRecord rec) throws IgniteCheckedException { FileWriteHandle hnd = currentHandle(); if (hnd != cur) @@ -1196,6 +1232,12 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I next.writeHeader(); + if (rec != null) { + WALPointer ptr = next.addRecord(rec); + + assert ptr != null; + } + if (next.getSegmentId() - lashCheckpointFileIdx() >= maxSegCountWithoutCheckpoint) cctx.database().forceCheckpoint("too big size of WAL without checkpoint"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index fc1461a215e74..7bfd10bba2376 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -77,6 +78,7 @@ import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; @@ -547,7 +549,7 @@ private void checkWalConfiguration() throws IgniteCheckedException { "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() + ", oldVer=" + currentHnd.serializer.version() + ']'); - rollOver(currentHnd); + rollOver(currentHnd, null); } if (mode == WALMode.BACKGROUND) { @@ -679,7 +681,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { final FileWriteHandle handle = currentHandle(); try { - rollOver(handle); + rollOver(handle, null); } catch (IgniteCheckedException e) { U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e); @@ -689,8 +691,12 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { } /** {@inheritDoc} */ - @SuppressWarnings("TooBroadScope") - @Override public WALPointer log(WALRecord record) throws IgniteCheckedException, StorageException { + @Override public WALPointer log(WALRecord record) throws IgniteCheckedException { + return log(record, RolloverType.NONE); + } + + /** {@inheritDoc} */ + @Override public WALPointer log(WALRecord record, RolloverType rolloverType) throws IgniteCheckedException { if (serializer == null || mode == WALMode.NONE) return null; @@ -706,13 +712,31 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { record.size(serializer.size(record)); while (true) { - if (record.rollOver()){ + WALPointer ptr; + + if (rolloverType == RolloverType.NONE) + ptr = currWrHandle.addRecord(record); + else { assert cctx.database().checkpointLockIsHeldByThread(); - currWrHandle = rollOver(currWrHandle); - } + if (rolloverType == RolloverType.NEXT_SEGMENT) { + WALPointer pos = record.position(); - WALPointer ptr = currWrHandle.addRecord(record); + do { + // This will change record.position() unless concurrent rollover happened. + currWrHandle = rollOver(currWrHandle, record); + } + while (Objects.equals(pos, record.position())); + + ptr = record.position(); + } + else if (rolloverType == RolloverType.CURRENT_SEGMENT) { + if ((ptr = currWrHandle.addRecord(record)) != null) + currWrHandle = rollOver(currWrHandle, null); + } + else + throw new IgniteCheckedException("Unknown rollover type: " + rolloverType); + } if (ptr != null) { metrics.onWalRecordLogged(); @@ -725,7 +749,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { return ptr; } else - currWrHandle = rollOver(currWrHandle); + currWrHandle = rollOver(currWrHandle, null); checkNode(); @@ -1115,9 +1139,10 @@ private FileWriteHandle currentHandle() { /** * @param cur Handle that failed to fit the given entry. + * @param rec Optional record to be added to the beginning of the segment. * @return Handle that will fit the entry. */ - private FileWriteHandle rollOver(FileWriteHandle cur) throws IgniteCheckedException { + private FileWriteHandle rollOver(FileWriteHandle cur, @Nullable WALRecord rec) throws IgniteCheckedException { FileWriteHandle hnd = currentHandle(); if (hnd != cur) @@ -1129,6 +1154,12 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws IgniteCheckedExcept FileWriteHandle next = initNextWriteHandle(cur.getSegmentId()); + if (rec != null) { + WALPointer ptr = next.addRecord(rec); + + assert ptr != null; + } + if (next.getSegmentId() - lashCheckpointFileIdx() >= maxSegCountWithoutCheckpoint) cctx.database().forceCheckpoint("too big size of WAL without checkpoint"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java index 395b03ae4367c..df6f57ddb1b8e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -51,11 +52,6 @@ private static class RolloverRecord extends CheckpointRecord { private RolloverRecord() { super(null); } - - /** {@inheritDoc} */ - @Override public boolean rollOver() { - return true; - } } /** {@inheritDoc} */ @@ -142,7 +138,7 @@ public void testAvoidInfinityWaitingOnRolloverOfSegment() throws Exception { dbMgr.checkpointReadLock(); try { - walMgr.log(rec); + walMgr.log(rec, RolloverType.NEXT_SEGMENT); } finally { dbMgr.checkpointReadUnlock(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverTypesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverTypesTest.java new file mode 100644 index 0000000000000..122ecb680d7bd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverTypesTest.java @@ -0,0 +1,366 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ignite.internal.processors.cache.persistence.db.wal; + +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.checkpoint.noop.NoopCheckpointSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH; +import static org.apache.ignite.configuration.WALMode.FSYNC; +import static org.apache.ignite.configuration.WALMode.LOG_ONLY; +import static org.apache.ignite.internal.pagemem.wal.record.RolloverType.CURRENT_SEGMENT; +import static org.apache.ignite.internal.pagemem.wal.record.RolloverType.NEXT_SEGMENT; +import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE; + +/** + * + */ +public class WalRolloverTypesTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private WALMode walMode; + + /** */ + private boolean disableWALArchiving; + + /** */ + private static class AdHocWALRecord extends CheckpointRecord { + /** */ + private AdHocWALRecord() { + super(null); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String name) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(name); + + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(20 * 1024 * 1024)) + .setWalMode(walMode) + .setWalArchivePath(disableWALArchiving ? DFLT_WAL_PATH : DFLT_WAL_ARCHIVE_PATH) + .setWalSegmentSize(4 * 1024 * 1024)) + .setCheckpointSpi(new NoopCheckpointSpi()) + ; + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** */ + public void testCurrentSegmentTypeLogOnlyModeArchiveOn() throws Exception { + checkCurrentSegmentType(LOG_ONLY, false); + } + + /** */ + public void testCurrentSegmentTypeLogOnlyModeArchiveOff() throws Exception { + checkCurrentSegmentType(LOG_ONLY, true); + } + + /** */ + public void testCurrentSegmentTypeLogFsyncModeArchiveOn() throws Exception { + checkCurrentSegmentType(FSYNC, false); + } + + /** */ + public void testCurrentSegmentTypeLogFsyncModeArchiveOff() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9776"); + + checkCurrentSegmentType(FSYNC, true); + } + + /** */ + public void testNextSegmentTypeLogOnlyModeArchiveOn() throws Exception { + checkNextSegmentType(LOG_ONLY, false); + } + + /** */ + public void testNextSegmentTypeLogOnlyModeArchiveOff() throws Exception { + checkNextSegmentType(LOG_ONLY, true); + } + + /** */ + public void testNextSegmentTypeFsyncModeArchiveOn() throws Exception { + checkNextSegmentType(FSYNC, false); + } + + /** */ + public void testNextSegmentTypeFsyncModeArchiveOff() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9776"); + + checkNextSegmentType(FSYNC, true); + } + + /** */ + private void checkCurrentSegmentType(WALMode mode, boolean disableArch) throws Exception { + walMode = mode; + disableWALArchiving = disableArch; + + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal(); + + ig.context().cache().context().database().checkpointReadLock(); + + try { + WALPointer ptr = walMgr.log(new AdHocWALRecord(), CURRENT_SEGMENT); + + assertEquals(0, ((FileWALPointer)ptr).index()); + } + finally { + ig.context().cache().context().database().checkpointReadUnlock(); + } + } + + /** */ + private void checkNextSegmentType(WALMode mode, boolean disableArch) throws Exception { + walMode = mode; + disableWALArchiving = disableArch; + + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal(); + + ig.context().cache().context().database().checkpointReadLock(); + + try { + WALPointer ptr = walMgr.log(new AdHocWALRecord(), NEXT_SEGMENT); + + assertEquals(1, ((FileWALPointer)ptr).index()); + } + finally { + ig.context().cache().context().database().checkpointReadUnlock(); + } + } + + /** */ + public void testNextSegmentTypeWithCacheActivityLogOnlyModeArchiveOn() throws Exception { + checkNextSegmentTypeWithCacheActivity(LOG_ONLY, false); + } + + /** */ + public void testNextSegmentTypeWithCacheActivityLogOnlyModeArchiveOff() throws Exception { + checkNextSegmentTypeWithCacheActivity(LOG_ONLY, true); + } + + /** */ + public void testNextSegmentTypeWithCacheActivityFsyncModeArchiveOn() throws Exception { + checkNextSegmentTypeWithCacheActivity(FSYNC, false); + } + + /** */ + public void testNextSegmentTypeWithCacheActivityFsyncModeArchiveOff() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9776"); + + checkNextSegmentTypeWithCacheActivity(FSYNC, true); + } + + /** + * Under load, ensures the record gets into very beginning of the segment in {@code NEXT_SEGMENT} log mode. + */ + private void checkNextSegmentTypeWithCacheActivity(WALMode mode, boolean disableArch) throws Exception { + walMode = mode; + disableWALArchiving = disableArch; + + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCache cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME); + + final long testDuration = 30_000; + + long startTime = U.currentTimeMillis(); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync( + () -> { + ThreadLocalRandom random = ThreadLocalRandom.current(); + + while (U.currentTimeMillis() - startTime < testDuration) + cache.put(random.nextInt(100), random.nextInt(100_000)); + }, + 8, "cache-put-thread"); + + IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal(); + + IgniteCacheDatabaseSharedManager dbMgr = ig.context().cache().context().database(); + + AdHocWALRecord markerRecord = new AdHocWALRecord(); + + WALPointer ptr0; + WALPointer ptr1; + + do { + try { + U.sleep(1000); + + ptr0 = walMgr.log(markerRecord); + + dbMgr.checkpointReadLock(); + + try { + ptr1 = walMgr.log(markerRecord, NEXT_SEGMENT); + } + finally { + dbMgr.checkpointReadUnlock(); + } + + assertTrue(ptr0 instanceof FileWALPointer); + assertTrue(ptr1 instanceof FileWALPointer); + + assertTrue(((FileWALPointer)ptr0).index() < ((FileWALPointer)ptr1).index()); + + assertEquals(HEADER_RECORD_SIZE, ((FileWALPointer)ptr1).fileOffset()); + } + catch (IgniteCheckedException e) { + log.error(e.getMessage(), e); + } + } + while (U.currentTimeMillis() - startTime < testDuration); + + fut.get(); + } + + /** */ + public void testCurrentSegmentTypeWithCacheActivityLogOnlyModeArchiveOn() throws Exception { + checkCurrentSegmentTypeWithCacheActivity(LOG_ONLY, false); + } + + /** */ + public void testCurrentSegmentTypeWithCacheActivityLogOnlyModeArchiveOff() throws Exception { + checkCurrentSegmentTypeWithCacheActivity(LOG_ONLY, true); + } + + /** */ + public void testCurrentSegmentTypeWithCacheActivityFsyncModeArchiveOn() throws Exception { + checkCurrentSegmentTypeWithCacheActivity(FSYNC, false); + } + + /** */ + public void testCurrentSegmentTypeWithCacheActivityFsyncModeArchiveOff() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9776"); + + checkCurrentSegmentTypeWithCacheActivity(FSYNC, true); + } + + /** + * Under load, ensures the record gets into very beginning of the segment in {@code NEXT_SEGMENT} log mode. + */ + private void checkCurrentSegmentTypeWithCacheActivity(WALMode mode, boolean disableArch) throws Exception { + walMode = mode; + disableWALArchiving = disableArch; + + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCache cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME); + + final long testDuration = 30_000; + + long startTime = U.currentTimeMillis(); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync( + () -> { + ThreadLocalRandom random = ThreadLocalRandom.current(); + + while (U.currentTimeMillis() - startTime < testDuration) + cache.put(random.nextInt(100), random.nextInt(100_000)); + }, + 8, "cache-put-thread"); + + IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal(); + + IgniteCacheDatabaseSharedManager dbMgr = ig.context().cache().context().database(); + + AdHocWALRecord markerRecord = new AdHocWALRecord(); + + WALPointer ptr0; + WALPointer ptr1; + + do { + try { + U.sleep(1000); + + dbMgr.checkpointReadLock(); + + try { + ptr0 = walMgr.log(markerRecord, CURRENT_SEGMENT); + } + finally { + dbMgr.checkpointReadUnlock(); + } + + ptr1 = walMgr.log(markerRecord); + + assertTrue(ptr0 instanceof FileWALPointer); + assertTrue(ptr1 instanceof FileWALPointer); + + assertTrue(((FileWALPointer)ptr0).index() < ((FileWALPointer)ptr1).index()); + } + catch (IgniteCheckedException e) { + log.error(e.getMessage(), e); + } + } + while (U.currentTimeMillis() - startTime < testDuration); + + fut.get(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java index df894190c3e00..0beeed783a007 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java @@ -20,12 +20,12 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; -import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor; +import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.lang.IgniteFuture; /** @@ -57,6 +57,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { return null; } + /** {@inheritDoc} */ + @Override public WALPointer log(WALRecord entry, RolloverType rollOverType) { + return null; + } + /** {@inheritDoc} */ @Override public void flush(WALPointer ptr, boolean explicitFsync) throws IgniteCheckedException, StorageException { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java index cf660c8b86832..4d2bdcfcc82fb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; @@ -108,7 +109,7 @@ public void testCorrectClosingFileDescriptors() throws Exception { sharedMgr.checkpointReadLock(); try { - walMgr.log(new SnapshotRecord(i, false)); + walMgr.log(new SnapshotRecord(i, false), RolloverType.NEXT_SEGMENT); } finally { sharedMgr.checkpointReadUnlock(); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index 7631834f9fc66..bf83bf55dc1e7 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalDeletionArchiveFsyncTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalDeletionArchiveLogOnlyTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverTypesTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteFsyncReplayWalIteratorInvalidCrcTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest; @@ -186,5 +187,7 @@ public static void addRealPageStoreTests(TestSuite suite) { suite.addTestSuite(IgniteRebalanceScheduleResendPartitionsTest.class); suite.addTestSuite(IgniteWALTailIsReachedDuringIterationOverArchiveTest.class); + + suite.addTestSuite(WalRolloverTypesTest.class); } } From 04e4ddc863598c4ad409bbd8c69109a1a1dbdf96 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 11 Oct 2018 19:00:27 +0300 Subject: [PATCH 016/403] IGNITE-9561 Optimize affinity initialization for started cache groups - Fixes #4854. (cherry picked from commit 5925444df9170652d4436288f470bf6d031c104e) --- .../cache/CacheAffinitySharedManager.java | 220 ++++++++++-------- .../cache/CacheGroupDescriptor.java | 19 ++ .../preloader/CacheGroupAffinityMessage.java | 12 +- .../GridDhtPartitionsExchangeFuture.java | 121 ++++++---- .../ignite/internal/util/IgniteUtils.java | 79 ++++++- 5 files changed, 297 insertions(+), 154 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index b57530bc5173a..bdb3e58feb476 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -29,8 +29,10 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.affinity.AffinityFunction; import org.apache.ignite.cluster.ClusterNode; @@ -922,28 +924,48 @@ private void processCacheStartRequests( time = System.currentTimeMillis(); - Set gprs = new HashSet<>(); + initAffinityOnCacheGroupsStart(fut, exchActions, crd); - for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) { - int grpId = action.descriptor().groupId(); + if (log.isInfoEnabled()) + log.info("Affinity initialization for started caches performed in " + (System.currentTimeMillis() - time) + " ms."); + } - if (gprs.add(grpId)) { - if (crd) - initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor()); - else { - CacheGroupContext grp = cctx.cache().cacheGroup(grpId); + /** + * Initializes affinity for started cache groups received during {@code fut}. + * + * @param fut Exchange future. + * @param exchangeActions Exchange actions. + * @param crd {@code True} if local node is coordinator. + */ + private void initAffinityOnCacheGroupsStart( + GridDhtPartitionsExchangeFuture fut, + ExchangeActions exchangeActions, + boolean crd + ) throws IgniteCheckedException { + List startedGroups = exchangeActions.cacheStartRequests().stream() + .map(action -> action.descriptor().groupDescriptor()) + .distinct() + .collect(Collectors.toList()); + + U.doInParallel( + cctx.kernalContext().getSystemExecutorService(), + startedGroups, + new IgniteInClosureX() { + @Override public void applyx(CacheGroupDescriptor grpDesc) throws IgniteCheckedException { + if (crd) + initStartedGroupOnCoordinator(fut, grpDesc); + else { + CacheGroupContext grp = cctx.cache().cacheGroup(grpDesc.groupId()); - if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) { - assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion(); + if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) { + assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion(); - initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut); + initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut); + } } } - } - } - - if (log.isInfoEnabled()) - log.info("Affinity initialization for started caches performed in " + (System.currentTimeMillis() - time) + " ms."); + }, + null); } /** @@ -952,7 +974,7 @@ private void processCacheStartRequests( * @param fut Exchange future. * @param crd Coordinator flag. * @param exchActions Cache change requests. - * @param forceClose + * @param forceClose Force close flag. * @return Set of cache groups to be stopped. */ private Set processCacheStopRequests( @@ -1009,9 +1031,11 @@ public void clearGroupHoldersAndRegistry() { * @param crd Coordinator flag. * @param msg Affinity change message. */ - public void onExchangeChangeAffinityMessage(GridDhtPartitionsExchangeFuture exchFut, + public void onExchangeChangeAffinityMessage( + GridDhtPartitionsExchangeFuture exchFut, boolean crd, - CacheAffinityChangeMessage msg) { + CacheAffinityChangeMessage msg + ) { if (log.isDebugEnabled()) { log.debug("Process exchange affinity change message [exchVer=" + exchFut.initialVersion() + ", msg=" + msg + ']'); @@ -1025,7 +1049,7 @@ public void onExchangeChangeAffinityMessage(GridDhtPartitionsExchangeFuture exch assert assignment != null; - final Map>> affCache = new HashMap<>(); + final Map>> affCache = new ConcurrentHashMap<>(); forAllCacheGroups(crd, new IgniteInClosureX() { @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException { @@ -1059,10 +1083,11 @@ public void onExchangeChangeAffinityMessage(GridDhtPartitionsExchangeFuture exch * @param msg Message. * @throws IgniteCheckedException If failed. */ - public void onChangeAffinityMessage(final GridDhtPartitionsExchangeFuture exchFut, + public void onChangeAffinityMessage( + final GridDhtPartitionsExchangeFuture exchFut, boolean crd, - final CacheAffinityChangeMessage msg) - throws IgniteCheckedException { + final CacheAffinityChangeMessage msg + ) { assert msg.topologyVersion() != null && msg.exchangeId() == null : msg; final AffinityTopologyVersion topVer = exchFut.initialVersion(); @@ -1078,7 +1103,7 @@ public void onChangeAffinityMessage(final GridDhtPartitionsExchangeFuture exchFu final Map deploymentIds = msg.cacheDeploymentIds(); - final Map>> affCache = new HashMap<>(); + final Map>> affCache = new ConcurrentHashMap<>(); forAllCacheGroups(crd, new IgniteInClosureX() { @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException { @@ -1190,14 +1215,17 @@ private void processAffinityAssignmentResponse(UUID nodeId, GridDhtAffinityAssig /** * @param c Cache closure. - * @throws IgniteCheckedException If failed */ - private void forAllRegisteredCacheGroups(IgniteInClosureX c) throws IgniteCheckedException { - for (CacheGroupDescriptor cacheDesc : cachesRegistry.allGroups().values()) { - if (cacheDesc.config().getCacheMode() == LOCAL) - continue; + private void forAllRegisteredCacheGroups(IgniteInClosureX c) { + Collection affinityCaches = cachesRegistry.allGroups().values().stream() + .filter(desc -> desc.config().getCacheMode() != LOCAL) + .collect(Collectors.toList()); - c.applyx(cacheDesc); + try { + U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null); + } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to execute affinity operation on cache groups", e); } } @@ -1206,17 +1234,25 @@ private void forAllRegisteredCacheGroups(IgniteInClosureX * @param c Closure. */ private void forAllCacheGroups(boolean crd, IgniteInClosureX c) { + Collection affinityCaches; + if (crd) { - for (CacheGroupHolder grp : grpHolders.values()) - c.apply(grp.affinity()); + affinityCaches = grpHolders.values().stream() + .map(CacheGroupHolder::affinity) + .collect(Collectors.toList()); } else { - for (CacheGroupContext grp : cctx.kernalContext().cache().cacheGroups()) { - if (grp.isLocal()) - continue; + affinityCaches = cctx.kernalContext().cache().cacheGroups().stream() + .filter(grp -> !grp.isLocal()) + .map(CacheGroupContext::affinity) + .collect(Collectors.toList()); + } - c.apply(grp.affinity()); - } + try { + U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null); + } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to execute affinity operation on cache groups", e); } } @@ -1372,11 +1408,14 @@ public GridAffinityAssignmentCache affinity(Integer grpId) { * @param fut Current exchange future. * @param msg Finish exchange message. */ - public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture fut, - final GridDhtPartitionsFullMessage msg) { - final Map nodesByOrder = new HashMap<>(); + public void applyAffinityFromFullMessage( + final GridDhtPartitionsExchangeFuture fut, + final GridDhtPartitionsFullMessage msg + ) { + // Please do not use following pattern of code (nodesByOrder, affCache). NEVER. + final Map nodesByOrder = new ConcurrentHashMap<>(); - final Map>> affCache = new HashMap<>(); + final Map>> affCache = new ConcurrentHashMap<>(); long time = System.currentTimeMillis(); @@ -1423,13 +1462,13 @@ public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture f * @param msg Message finish message. * @param resTopVer Result topology version. */ - public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut, + public void onLocalJoin( + final GridDhtPartitionsExchangeFuture fut, GridDhtPartitionsFullMessage msg, - final AffinityTopologyVersion resTopVer) { + final AffinityTopologyVersion resTopVer + ) { final Set affReq = fut.context().groupsAffinityRequestOnJoin(); - final Map nodesByOrder = new HashMap<>(); - final Map receivedAff = msg.joinedNodeAffinity(); assert F.isEmpty(affReq) || (!F.isEmpty(receivedAff) && receivedAff.size() >= affReq.size()) @@ -1438,6 +1477,8 @@ public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut, ", receivedCnt=" + (receivedAff != null ? receivedAff.size() : "none") + ", msg=" + msg + "]"); + final Map nodesByOrder = new ConcurrentHashMap<>(); + long time = System.currentTimeMillis(); forAllCacheGroups(false, new IgniteInClosureX() { @@ -1781,8 +1822,8 @@ private GridDhtAffinityAssignmentResponse fetchAffinity( @Nullable ExchangeDiscoveryEvents events, DiscoCache discoCache, GridAffinityAssignmentCache affCache, - GridDhtAssignmentFetchFuture fetchFut) - throws IgniteCheckedException { + GridDhtAssignmentFetchFuture fetchFut + ) throws IgniteCheckedException { assert affCache != null; GridDhtAffinityAssignmentResponse res = fetchFut.get(); @@ -1860,7 +1901,7 @@ public IgniteInternalFuture initCoordinatorCaches( final GridDhtPartitionsExchangeFuture fut, final boolean newAff ) throws IgniteCheckedException { - final List> futs = new ArrayList<>(); + final List> futs = Collections.synchronizedList(new ArrayList<>()); final AffinityTopologyVersion topVer = fut.initialVersion(); @@ -2013,29 +2054,28 @@ private CacheGroupHolder groupHolder(AffinityTopologyVersion topVer, final Cache /** * @param fut Current exchange future. * @param crd Coordinator flag. - * @throws IgniteCheckedException If failed. * @return Rabalance info. */ - @Nullable private WaitRebalanceInfo initAffinityOnNodeJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd) - throws IgniteCheckedException { + @Nullable private WaitRebalanceInfo initAffinityOnNodeJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd) { final ExchangeDiscoveryEvents evts = fut.context().events(); - final Map>> affCache = new HashMap<>(); + final Map>> affCache = new ConcurrentHashMap<>(); if (!crd) { - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { - if (grp.isLocal()) - continue; + forAllCacheGroups(false, new IgniteInClosureX() { + @Override public void applyx(GridAffinityAssignmentCache grpAffCache) throws IgniteCheckedException { + CacheGroupContext grp = cctx.cache().cacheGroup(grpAffCache.groupId()); - boolean latePrimary = grp.rebalanceEnabled(); + assert grp != null; - initAffinityOnNodeJoin(evts, - evts.nodeJoined(grp.receivedFrom()), - grp.affinity(), - null, - latePrimary, - affCache); - } + initAffinityOnNodeJoin(evts, + evts.nodeJoined(grp.receivedFrom()), + grp.affinity(), + null, + grp.rebalanceEnabled(), + affCache); + } + }); return null; } @@ -2077,6 +2117,9 @@ private CacheGroupHolder groupHolder(AffinityTopologyVersion topVer, final Cache } } + /** + * @param aff Affinity assignment. + */ private Map affinityFullMap(AffinityAssignment aff) { Map map = new HashMap<>(); @@ -2242,7 +2285,7 @@ public IgniteInternalFuture>>> initAffinity try { resFut.onDone(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false)); } - catch (IgniteCheckedException e) { + catch (Exception e) { resFut.onDone(e); } } @@ -2262,14 +2305,15 @@ public IgniteInternalFuture>>> initAffinity * @param fut Exchange future. * @param c Closure converting affinity diff. * @param initAff {@code True} if need initialize affinity. - * @return Affinity assignment. - * @throws IgniteCheckedException If failed. + * + * @return Affinity assignment for each of registered cache group. */ - private Map>> initAffinityBasedOnPartitionsAvailability(final AffinityTopologyVersion topVer, + private Map>> initAffinityBasedOnPartitionsAvailability( + final AffinityTopologyVersion topVer, final GridDhtPartitionsExchangeFuture fut, final IgniteClosure c, - final boolean initAff) - throws IgniteCheckedException { + final boolean initAff + ) { final boolean enforcedCentralizedAssignment = DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent()); @@ -2279,7 +2323,7 @@ private Map>> initAffinityBasedOnPartitionsAva final Collection aliveNodes = fut.context().events().discoveryCache().serverNodes(); - final Map>> assignment = new HashMap<>(); + final Map>> assignment = new ConcurrentHashMap<>(); forAllRegisteredCacheGroups(new IgniteInClosureX() { @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException { @@ -2675,13 +2719,13 @@ class WaitRebalanceInfo { private final AffinityTopologyVersion topVer; /** */ - private Map> waitGrps; + private final Map> waitGrps = new ConcurrentHashMap<>(); /** */ - private Map>> assignments; + private final Map>> assignments = new ConcurrentHashMap<>(); /** */ - private Map deploymentIds; + private final Map deploymentIds = new ConcurrentHashMap<>(); /** * @param topVer Topology version. @@ -2694,14 +2738,15 @@ class WaitRebalanceInfo { * @return {@code True} if there are partitions waiting for rebalancing. */ boolean empty() { - if (waitGrps != null) { - assert !waitGrps.isEmpty(); + boolean isEmpty = waitGrps.isEmpty(); + + if (!isEmpty) { assert waitGrps.size() == assignments.size(); return false; } - return true; + return isEmpty; } /** @@ -2713,28 +2758,11 @@ boolean empty() { void add(Integer grpId, Integer part, UUID waitNode, List assignment) { assert !F.isEmpty(assignment) : assignment; - if (waitGrps == null) { - waitGrps = new HashMap<>(); - assignments = new HashMap<>(); - deploymentIds = new HashMap<>(); - } - - Map cacheWaitParts = waitGrps.get(grpId); - - if (cacheWaitParts == null) { - waitGrps.put(grpId, cacheWaitParts = new HashMap<>()); - - deploymentIds.put(grpId, cachesRegistry.group(grpId).deploymentId()); - } - - cacheWaitParts.put(part, waitNode); - - Map> cacheAssignment = assignments.get(grpId); + deploymentIds.putIfAbsent(grpId, cachesRegistry.group(grpId).deploymentId()); - if (cacheAssignment == null) - assignments.put(grpId, cacheAssignment = new HashMap<>()); + waitGrps.computeIfAbsent(grpId, k -> new HashMap<>()).put(part, waitNode); - cacheAssignment.put(part, assignment); + assignments.computeIfAbsent(grpId, k -> new HashMap<>()).put(part, assignment); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java index 70cdcc735af62..e72de28a05674 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java @@ -23,6 +23,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; @@ -311,4 +312,22 @@ public boolean persistenceEnabled() { @Override public String toString() { return S.toString(CacheGroupDescriptor.class, this, "cacheName", cacheCfg.getName()); } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + CacheGroupDescriptor that = (CacheGroupDescriptor) o; + + return grpId == that.grpId; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(grpId); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java index 7da4051e27929..695eadca581a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java @@ -192,16 +192,10 @@ public static List toNodes(GridLongList assign, Map discoCache.serverNodeByOrder(order)); - if (affNode == null) { - affNode = discoCache.serverNodeByOrder(order); - - assert affNode != null : "Failed to find node by order [order=" + order + - ", topVer=" + discoCache.version() + ']'; - - nodesByOrder.put(order, affNode); - } + assert affNode != null : "Failed to find node by order [order=" + order + + ", topVer=" + discoCache.version() + ']'; assign0.add(affNode); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 514b81fb9a5b4..b2362e299f806 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheMode; @@ -85,13 +86,13 @@ import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext; import org.apache.ignite.internal.processors.cache.StateChangeRequest; import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch; import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; @@ -103,6 +104,7 @@ import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CI1; @@ -3271,44 +3273,60 @@ else if (forceAffReassignment) } } + /** + * Collects non local cache group descriptors. + * + * @return Collection of non local cache group descriptors. + */ + private List nonLocalCacheGroupDescriptors() { + return cctx.affinity().cacheGroups().values().stream() + .filter(grpDesc -> grpDesc.config().getCacheMode() != CacheMode.LOCAL) + .collect(Collectors.toList()); + } + /** * Validates that partition update counters and cache sizes for all caches are consistent. */ private void validatePartitionsState() { long time = System.currentTimeMillis(); - for (Map.Entry e : cctx.affinity().cacheGroups().entrySet()) { - CacheGroupDescriptor grpDesc = e.getValue(); - if (grpDesc.config().getCacheMode() == CacheMode.LOCAL) - continue; - - int grpId = e.getKey(); - - CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpId); - - GridDhtPartitionTopology top = grpCtx != null ? - grpCtx.topology() : - cctx.exchange().clientTopology(grpId, events().discoveryCache()); - - // Do not validate read or write through caches or caches with disabled rebalance - // or ExpiryPolicy is set or validation is disabled. - if (grpCtx == null - || grpCtx.config().isReadThrough() - || grpCtx.config().isWriteThrough() - || grpCtx.config().getCacheStoreFactory() != null - || grpCtx.config().getRebalanceDelay() == -1 - || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE - || grpCtx.config().getExpiryPolicyFactory() == null - || SKIP_PARTITION_SIZE_VALIDATION) - continue; + try { + U.doInParallel( + cctx.kernalContext().getSystemExecutorService(), + nonLocalCacheGroupDescriptors(), + new IgniteInClosureX() { + @Override public void applyx(CacheGroupDescriptor grpDesc) { + CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId()); + + GridDhtPartitionTopology top = grpCtx != null + ? grpCtx.topology() + : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache()); + + // Do not validate read or write through caches or caches with disabled rebalance + // or ExpiryPolicy is set or validation is disabled. + if (grpCtx == null + || grpCtx.config().isReadThrough() + || grpCtx.config().isWriteThrough() + || grpCtx.config().getCacheStoreFactory() != null + || grpCtx.config().getRebalanceDelay() == -1 + || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE + || grpCtx.config().getExpiryPolicyFactory() == null + || SKIP_PARTITION_SIZE_VALIDATION) + return; - try { - validator.validatePartitionCountersAndSizes(this, top, msgs); - } - catch (IgniteCheckedException ex) { - log.warning("Partition states validation has failed for group: " + grpDesc.cacheOrGroupName() + ". " + ex.getMessage()); - // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833 - } + try { + validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs); + } + catch (IgniteCheckedException ex) { + log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage()); + // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833 + } + } + }, + null); + } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to validate partitions state", e); } if (log.isInfoEnabled()) @@ -3321,21 +3339,28 @@ private void validatePartitionsState() { private void assignPartitionsStates() { long time = System.currentTimeMillis(); - for (Map.Entry e : cctx.affinity().cacheGroups().entrySet()) { - CacheGroupDescriptor grpDesc = e.getValue(); - if (grpDesc.config().getCacheMode() == CacheMode.LOCAL) - continue; - - CacheGroupContext grpCtx = cctx.cache().cacheGroup(e.getKey()); - - GridDhtPartitionTopology top = grpCtx != null ? - grpCtx.topology() : - cctx.exchange().clientTopology(e.getKey(), events().discoveryCache()); - - if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration())) - assignPartitionSizes(top); - else - assignPartitionStates(top); + try { + U.doInParallel( + cctx.kernalContext().getSystemExecutorService(), + nonLocalCacheGroupDescriptors(), + new IgniteInClosureX() { + @Override public void applyx(CacheGroupDescriptor grpDesc) { + CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId()); + + GridDhtPartitionTopology top = grpCtx != null + ? grpCtx.topology() + : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache()); + + if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration())) + assignPartitionSizes(top); + else + assignPartitionStates(top); + } + }, + null); + } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to assign partition states", e); } if (log.isInfoEnabled()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 5f397d53f8867..8dbea170e9cfb 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -141,6 +141,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; @@ -212,16 +213,19 @@ import org.apache.ignite.internal.util.lang.GridClosureException; import org.apache.ignite.internal.util.lang.GridPeerDeployAware; import org.apache.ignite.internal.util.lang.GridTuple; +import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.typedef.C1; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.P1; +import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.lang.IgniteFutureCancelledException; @@ -10354,7 +10358,7 @@ public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException { public static byte[] toBytes(Serializable obj) { try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(bos)) { - + oos.writeObject(obj); oos.flush(); @@ -10644,6 +10648,79 @@ public static ClusterNode randomServerNode(GridKernalContext ctx) { return rndNode; } + /** + * @param executorSvc Service for parallel execution. + * @param srcDatas List of data for parallelization. + * @param consumer Logic for execution of on each item of data. + * @param errHnd Optionan error handler. If not {@code null}, an error of each item execution will be passed to + * this handler. If error handler is not {@code null}, the exception will not be thrown from this method. + * @param Type of data. + * @return List of (item, execution future) tuples. + * @throws IgniteCheckedException If parallel execution failed and {@code errHnd} is {@code null}. + */ + public static List>> doInParallel( + ExecutorService executorSvc, + Collection srcDatas, + IgniteInClosureX consumer, + @Nullable IgniteBiInClosure errHnd + ) throws IgniteCheckedException { + List>> consumerFutures = srcDatas.stream() + .map(item -> new T2<>( + item, + executorSvc.submit(() -> { + consumer.apply(item); + + return null; + }))) + .collect(Collectors.toList()); + + IgniteCheckedException composite = null; + + for (T2> tup : consumerFutures) { + try { + getUninterruptibly(tup.get2()); + } + catch (ExecutionException e) { + if (errHnd != null) + errHnd.apply(tup.get1(), e.getCause()); + else { + if (composite == null) + composite = new IgniteCheckedException("Failed to execute one of the tasks " + + "(see suppressed exception for details)"); + + composite.addSuppressed(e.getCause()); + } + } + } + + if (composite != null) + throw composite; + + return consumerFutures; + } + + /** + * @param fut Future to wait for completion. + * @throws ExecutionException If the future + */ + private static void getUninterruptibly(Future fut) throws ExecutionException { + boolean interrupted = false; + + while (true) { + try { + fut.get(); + + break; + } + catch (InterruptedException e) { + interrupted = true; + } + } + + if (interrupted) + Thread.currentThread().interrupt(); + } + /** * */ From db29b8c81c85cadb38538aee55a6e01f156cdca4 Mon Sep 17 00:00:00 2001 From: Aleksei Scherbakov Date: Wed, 17 Oct 2018 13:28:20 +0300 Subject: [PATCH 017/403] IGNITE-9830 Fixed race in binary metadata registration leading to exception on commit - Fixes #4996. Signed-off-by: Alexey Goncharuk --- .../apache/ignite/IgniteSystemProperties.java | 11 + .../internal/binary/BinaryReaderExImpl.java | 1 - .../internal/binary/BinarySchemaRegistry.java | 114 +++-- .../ignite/internal/binary/BinaryUtils.java | 28 +- .../cache/binary/BinaryMetadataTransport.java | 93 +++- .../CacheObjectBinaryProcessorImpl.java | 219 ++++++++- .../spi/discovery/tcp/TcpDiscoverySpi.java | 5 +- .../CachePageWriteLockUnlockTest.java | 2 + .../transactions/TxRollbackOnTimeoutTest.java | 7 +- ...tadataConcurrentUpdateWithIndexesTest.java | 439 ++++++++++++++++++ .../IgniteBinaryCacheQueryTestSuite.java | 3 + 11 files changed, 833 insertions(+), 89 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 81b767f74ec2c..02ebb25731dd9 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1007,6 +1007,17 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_REUSE_MEMORY_ON_DEACTIVATE = "IGNITE_REUSE_MEMORY_ON_DEACTIVATE"; + /** + * Timeout for waiting schema update if schema was not found for last accepted version. + */ + public static final String IGNITE_WAIT_SCHEMA_UPDATE = "IGNITE_WAIT_SCHEMA_UPDATE"; + + /** + * System property to override {@link CacheConfiguration#rebalanceThrottle} configuration property for all caches. + * {@code 0} by default, which means that override is disabled. + */ + public static final String IGNITE_REBALANCE_THROTTLE_OVERRIDE = "IGNITE_REBALANCE_THROTTLE_OVERRIDE"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java index 38934f07d6ae0..601141c306416 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java @@ -2028,7 +2028,6 @@ public BinarySchema getOrCreateSchema() { for (BinarySchema existingSchema : existingSchemas) existingSchemaIds.add(existingSchema.schemaId()); - throw new BinaryObjectException("Cannot find schema for object with compact footer" + " [typeName=" + type.typeName() + ", typeId=" + typeId + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java index 91f29b22cfd8a..f22fc4c052121 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.binary; +import java.util.ArrayList; +import java.util.List; import org.jetbrains.annotations.Nullable; import java.util.HashMap; @@ -98,75 +100,95 @@ else if (schemaId == schemaId4) * @param schemaId Schema ID. * @param schema Schema. */ - public void addSchema(int schemaId, BinarySchema schema) { - synchronized (this) { - if (inline) { - // Check if this is already known schema. - if (schemaId == schemaId1 || schemaId == schemaId2 || schemaId == schemaId3 || schemaId == schemaId4) - return; + public synchronized void addSchema(int schemaId, BinarySchema schema) { + if (inline) { + // Check if this is already known schema. + if (schemaId == schemaId1 || schemaId == schemaId2 || schemaId == schemaId3 || schemaId == schemaId4) + return; - // Try positioning new schema in inline mode. - if (schemaId1 == EMPTY) { - schemaId1 = schemaId; + // Try positioning new schema in inline mode. + if (schemaId1 == EMPTY) { + schemaId1 = schemaId; - schema1 = schema; + schema1 = schema; - inline = true; // Forcing HB edge just in case. + inline = true; // Forcing HB edge just in case. - return; - } + return; + } - if (schemaId2 == EMPTY) { - schemaId2 = schemaId; + if (schemaId2 == EMPTY) { + schemaId2 = schemaId; - schema2 = schema; + schema2 = schema; - inline = true; // Forcing HB edge just in case. + inline = true; // Forcing HB edge just in case. - return; - } + return; + } - if (schemaId3 == EMPTY) { - schemaId3 = schemaId; + if (schemaId3 == EMPTY) { + schemaId3 = schemaId; - schema3 = schema; + schema3 = schema; - inline = true; // Forcing HB edge just in case. + inline = true; // Forcing HB edge just in case. - return; - } + return; + } - if (schemaId4 == EMPTY) { - schemaId4 = schemaId; + if (schemaId4 == EMPTY) { + schemaId4 = schemaId; - schema4 = schema; + schema4 = schema; - inline = true; // Forcing HB edge just in case. + inline = true; // Forcing HB edge just in case. - return; - } + return; + } - // No luck, switching to hash map mode. - HashMap newSchemas = new HashMap<>(); + // No luck, switching to hash map mode. + HashMap newSchemas = new HashMap<>(); - newSchemas.put(schemaId1, schema1); - newSchemas.put(schemaId2, schema2); - newSchemas.put(schemaId3, schema3); - newSchemas.put(schemaId4, schema4); + newSchemas.put(schemaId1, schema1); + newSchemas.put(schemaId2, schema2); + newSchemas.put(schemaId3, schema3); + newSchemas.put(schemaId4, schema4); - newSchemas.put(schemaId, schema); + newSchemas.put(schemaId, schema); - schemas = newSchemas; + schemas = newSchemas; - inline = false; - } - else { - HashMap newSchemas = new HashMap<>(schemas); + inline = false; + } + else { + HashMap newSchemas = new HashMap<>(schemas); - newSchemas.put(schemaId, schema); + newSchemas.put(schemaId, schema); - schemas = newSchemas; - } + schemas = newSchemas; } } + + /** + * @return List of known schemas. + */ + public synchronized List schemas() { + List res = new ArrayList<>(); + + if (inline) { + if (schemaId1 != EMPTY) + res.add(schema1); + if (schemaId2 != EMPTY) + res.add(schema2); + if (schemaId3 != EMPTY) + res.add(schema3); + if (schemaId4 != EMPTY) + res.add(schema4); + } + else + res.addAll(schemas.values()); + + return res; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 553d8e5b79bd9..77dce5602ec4e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -958,10 +958,30 @@ else if (fieldOffsetSize == OFFSET_2) * @throws BinaryObjectException If merge failed due to metadata conflict. */ public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, BinaryMetadata newMeta) { + return mergeMetadata(oldMeta, newMeta, null); + } + + /** + * Merge old and new metas. + * + * @param oldMeta Old meta. + * @param newMeta New meta. + * @param changedSchemas Set for holding changed schemas. + * @return New meta if old meta was null, old meta if no changes detected, merged meta otherwise. + * @throws BinaryObjectException If merge failed due to metadata conflict. + */ + public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, BinaryMetadata newMeta, + @Nullable Set changedSchemas) { assert newMeta != null; - if (oldMeta == null) + if (oldMeta == null) { + if (changedSchemas != null) { + for (BinarySchema schema : newMeta.schemas()) + changedSchemas.add(schema.schemaId()); + } + return newMeta; + } else { assert oldMeta.typeId() == newMeta.typeId(); @@ -1036,8 +1056,12 @@ public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, Bin Collection mergedSchemas = new HashSet<>(oldMeta.schemas()); for (BinarySchema newSchema : newMeta.schemas()) { - if (mergedSchemas.add(newSchema)) + if (mergedSchemas.add(newSchema)) { changed = true; + + if (changedSchemas != null) + changedSchemas.add(newSchema.schemaId()); + } } // Return either old meta if no changes detected, or new merged meta. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java index 38450dfec5c78..1c2f6f0e77f84 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java @@ -16,8 +16,12 @@ */ package org.apache.ignite.internal.processors.cache.binary; +import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; @@ -42,6 +46,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; import org.jetbrains.annotations.Nullable; @@ -86,6 +91,9 @@ final class BinaryMetadataTransport { /** */ private final ConcurrentMap clientReqSyncMap = new ConcurrentHashMap<>(); + /** */ + private final ConcurrentMap> schemaWaitFuts = new ConcurrentHashMap<>(); + /** */ private volatile boolean stopping; @@ -206,6 +214,21 @@ GridFutureAdapter awaitMetadataUpdate(int typeId, int ver) return resFut; } + /** + * Await specific schema update. + * @param typeId Type id. + * @param schemaId Schema id. + * @return Future which will be completed when schema is received. + */ + GridFutureAdapter awaitSchemaUpdate(int typeId, int schemaId) { + GridFutureAdapter fut = new GridFutureAdapter<>(); + + // Use version for schemaId. + GridFutureAdapter oldFut = schemaWaitFuts.putIfAbsent(new SyncKey(typeId, schemaId), fut); + + return oldFut == null ? fut : oldFut; + } + /** * Allows client node to request latest version of binary metadata for a given typeId from the cluster * in case client is able to detect that it has obsolete metadata in its local cache. @@ -259,6 +282,13 @@ private final class MetadataUpdateProposedListener implements CustomEventListene /** {@inheritDoc} */ @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, MetadataUpdateProposedMessage msg) { + if (log.isDebugEnabled()) + log.debug("Received MetadataUpdateProposedListener [typeId=" + msg.typeId() + + ", typeName=" + msg.metadata().typeName() + + ", pendingVer=" + msg.pendingVersion() + + ", acceptedVer=" + msg.acceptedVersion() + + ", schemasCnt=" + msg.metadata().schemas().size() + ']'); + int typeId = msg.typeId(); BinaryMetadataHolder holder = metaLocCache.get(typeId); @@ -277,20 +307,23 @@ private final class MetadataUpdateProposedListener implements CustomEventListene acceptedVer = 0; } - if (log.isDebugEnabled()) - log.debug("Versions are stamped on coordinator" + - " [typeId=" + typeId + - ", pendingVer=" + pendingVer + - ", acceptedVer=" + acceptedVer + "]" - ); - msg.pendingVersion(pendingVer); msg.acceptedVersion(acceptedVer); BinaryMetadata locMeta = holder != null ? holder.metadata() : null; try { - BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata()); + Set changedSchemas = new LinkedHashSet<>(); + + BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata(), changedSchemas); + + if (log.isDebugEnabled()) + log.debug("Versions are stamped on coordinator" + + " [typeId=" + typeId + + ", changedSchemas=" + changedSchemas + + ", pendingVer=" + pendingVer + + ", acceptedVer=" + acceptedVer + "]" + ); msg.metadata(mergedMeta); } @@ -358,8 +391,10 @@ private final class MetadataUpdateProposedListener implements CustomEventListene if (!msg.rejected()) { BinaryMetadata locMeta = holder != null ? holder.metadata() : null; + Set changedSchemas = new LinkedHashSet<>(); + try { - BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata()); + BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata(), changedSchemas); BinaryMetadataHolder newHolder = new BinaryMetadataHolder(mergedMeta, pendingVer, acceptedVer); @@ -382,7 +417,8 @@ private final class MetadataUpdateProposedListener implements CustomEventListene } else { if (log.isDebugEnabled()) - log.debug("Updated metadata on server node: " + newHolder); + log.debug("Updated metadata on server node [holder=" + newHolder + + ", changedSchemas=" + changedSchemas + ']'); metaLocCache.put(typeId, newHolder); } @@ -463,7 +499,7 @@ private final class MetadataUpdateAcceptedListener implements CustomEventListene if (oldAcceptedVer >= newAcceptedVer) { if (log.isDebugEnabled()) log.debug("Marking ack as duplicate [holder=" + holder + - ", newAcceptedVer: " + newAcceptedVer + ']'); + ", newAcceptedVer=" + newAcceptedVer + ']'); //this is duplicate ack msg.duplicated(true); @@ -481,8 +517,26 @@ private final class MetadataUpdateAcceptedListener implements CustomEventListene GridFutureAdapter fut = syncMap.get(new SyncKey(typeId, newAcceptedVer)); + holder = metaLocCache.get(typeId); + if (log.isDebugEnabled()) - log.debug("Completing future " + fut + " for " + metaLocCache.get(typeId)); + log.debug("Completing future " + fut + " for " + holder); + + if (!schemaWaitFuts.isEmpty()) { + Iterator>> iter = schemaWaitFuts.entrySet().iterator(); + + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + + SyncKey key = entry.getKey(); + + if (key.typeId() == typeId && holder.metadata().hasSchema(key.version())) { + entry.getValue().onDone(); + + iter.remove(); + } + } + } if (fut != null) fut.onDone(MetadataUpdateResult.createSuccessfulResult()); @@ -527,6 +581,11 @@ private final class MetadataUpdateResultFuture extends GridFutureAdapter changedSchemas = new LinkedHashSet<>(); + + BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0, changedSchemas); - //metadata requested to be added is exactly the same as already presented in the cache - if (mergedMeta == oldMeta) - return; + if (oldMeta != null && mergedMeta == oldMeta && metaHolder.pendingVersion() == metaHolder.acceptedVersion()) + return; // Safe to use existing schemas. if (failIfUnregistered) throw new UnregisteredBinaryTypeException( @@ -466,7 +490,24 @@ public GridBinaryMarshaller marshaller() { "dev-list.", typeId, mergedMeta); - MetadataUpdateResult res = transport.requestMetadataUpdate(mergedMeta).get(); + long t0 = System.nanoTime(); + + GridFutureAdapter fut = transport.requestMetadataUpdate(mergedMeta); + + MetadataUpdateResult res = fut.get(); + + if (log.isDebugEnabled()) { + IgniteInternalTx tx = ctx.cache().context().tm().tx(); + + log.debug("Completed metadata update [typeId=" + typeId + + ", typeName=" + newMeta.typeName() + + ", changedSchemas=" + changedSchemas + + ", waitTime=" + MILLISECONDS.convert(System.nanoTime() - t0, NANOSECONDS) + "ms" + + ", holder=" + metaHolder + + ", fut=" + fut + + ", tx=" + CU.txString(tx) + + ']'); + } assert res != null; @@ -541,9 +582,9 @@ public GridBinaryMarshaller marshaller() { if (log.isDebugEnabled() && !fut.isDone()) log.debug("Waiting for update for" + - " [typeId=" + typeId + - ", pendingVer=" + holder.pendingVersion() + - ", acceptedVer=" + holder.acceptedVersion() + "]"); + " [typeId=" + typeId + + ", pendingVer=" + holder.pendingVersion() + + ", acceptedVer=" + holder.acceptedVersion() + "]"); try { fut.get(); @@ -565,40 +606,99 @@ public GridBinaryMarshaller marshaller() { if (ctx.clientNode()) { if (holder == null || !holder.metadata().hasSchema(schemaId)) { + if (log.isDebugEnabled()) + log.debug("Waiting for client metadata update" + + " [typeId=" + typeId + + ", schemaId=" + schemaId + + ", pendingVer=" + (holder == null ? "NA" : holder.pendingVersion()) + + ", acceptedVer=" + (holder == null ? "NA" :holder.acceptedVersion()) + ']'); + try { transport.requestUpToDateMetadata(typeId).get(); - - holder = metadataLocCache.get(typeId); } catch (IgniteCheckedException ignored) { // No-op. } + + holder = metadataLocCache.get(typeId); + + if (log.isDebugEnabled()) + log.debug("Finished waiting for client metadata update" + + " [typeId=" + typeId + + ", schemaId=" + schemaId + + ", pendingVer=" + (holder == null ? "NA" : holder.pendingVersion()) + + ", acceptedVer=" + (holder == null ? "NA" :holder.acceptedVersion()) + ']'); } } - else if (holder != null) { - if (IgniteThread.current() instanceof IgniteDiscoveryThread) + else { + if (holder != null && IgniteThread.current() instanceof IgniteDiscoveryThread) return holder.metadata().wrap(binaryCtx); + else if (holder != null && (holder.pendingVersion() - holder.acceptedVersion() > 0)) { + if (log.isDebugEnabled()) + log.debug("Waiting for metadata update" + + " [typeId=" + typeId + + ", schemaId=" + schemaId + + ", pendingVer=" + holder.pendingVersion() + + ", acceptedVer=" + holder.acceptedVersion() + ']'); - if (holder.pendingVersion() - holder.acceptedVersion() > 0) { - GridFutureAdapter fut = transport.awaitMetadataUpdate( - typeId, - holder.pendingVersion()); + long t0 = System.nanoTime(); - if (log.isDebugEnabled() && !fut.isDone()) - log.debug("Waiting for update for" + - " [typeId=" + typeId - + ", schemaId=" + schemaId - + ", pendingVer=" + holder.pendingVersion() - + ", acceptedVer=" + holder.acceptedVersion() + "]"); + GridFutureAdapter fut = transport.awaitMetadataUpdate( + typeId, + holder.pendingVersion()); try { fut.get(); } + catch (IgniteCheckedException e) { + log.error("Failed to wait for metadata update [typeId=" + typeId + ", schemaId=" + schemaId + ']', e); + } + + if (log.isDebugEnabled()) + log.debug("Finished waiting for metadata update" + + " [typeId=" + typeId + + ", waitTime=" + NANOSECONDS.convert(System.nanoTime() - t0, MILLISECONDS) + "ms" + + ", schemaId=" + schemaId + + ", pendingVer=" + holder.pendingVersion() + + ", acceptedVer=" + holder.acceptedVersion() + ']'); + + holder = metadataLocCache.get(typeId); + } + else if (holder == null || !holder.metadata().hasSchema(schemaId)) { + // Last resort waiting. + U.warn(log, + "Schema is missing while no metadata updates are in progress " + + "(will wait for schema update within timeout defined by IGNITE_BINARY_META_UPDATE_TIMEOUT system property)" + + " [typeId=" + typeId + + ", missingSchemaId=" + schemaId + + ", pendingVer=" + (holder == null ? "NA" : holder.pendingVersion()) + + ", acceptedVer=" + (holder == null ? "NA" : holder.acceptedVersion()) + + ", binMetaUpdateTimeout=" + waitSchemaTimeout +']'); + + long t0 = System.nanoTime(); + + GridFutureAdapter fut = transport.awaitSchemaUpdate(typeId, schemaId); + + try { + fut.get(waitSchemaTimeout); + } + catch (IgniteFutureTimeoutCheckedException e) { + log.error("Timed out while waiting for schema update [typeId=" + typeId + ", schemaId=" + + schemaId + ']'); + } catch (IgniteCheckedException ignored) { // No-op. } holder = metadataLocCache.get(typeId); + + if (log.isDebugEnabled() && holder != null && holder.metadata().hasSchema(schemaId)) + log.debug("Found the schema after wait" + + " [typeId=" + typeId + + ", waitTime=" + NANOSECONDS.convert(System.nanoTime() - t0, MILLISECONDS) + "ms" + + ", schemaId=" + schemaId + + ", pendingVer=" + holder.pendingVersion() + + ", acceptedVer=" + holder.acceptedVersion() + ']'); } } @@ -903,7 +1003,7 @@ else if (type == BinaryObjectImpl.TYPE_BINARY_ENUM) if ((res = validateBinaryConfiguration(rmtNode)) != null) return res; - return validateBinaryMetadata(rmtNode.id(), (Map) discoData.joiningNodeData()); + return validateBinaryMetadata(rmtNode.id(), (Map)discoData.joiningNodeData()); } /** */ @@ -1070,4 +1170,75 @@ private IgniteNodeValidationResult validateBinaryMetadata(UUID rmtNodeId, Map listeners; + + /** + * @param metaHnd Meta handler. + * @param igniteCfg Ignite config. + * @param log Logger. + */ + public TestBinaryContext(BinaryMetadataHandler metaHnd, IgniteConfiguration igniteCfg, + IgniteLogger log) { + super(metaHnd, igniteCfg, log); + } + + /** {@inheritDoc} */ + @Nullable @Override public BinaryType metadata(int typeId) throws BinaryObjectException { + BinaryType metadata = super.metadata(typeId); + + if (listeners != null) { + for (TestBinaryContextListener listener : listeners) + listener.onAfterMetadataRequest(typeId, metadata); + } + + return metadata; + } + + /** {@inheritDoc} */ + @Override public void updateMetadata(int typeId, BinaryMetadata meta, + boolean failIfUnregistered) throws BinaryObjectException { + if (listeners != null) { + for (TestBinaryContextListener listener : listeners) + listener.onBeforeMetadataUpdate(typeId, meta); + } + + super.updateMetadata(typeId, meta, failIfUnregistered); + } + + /** */ + public interface TestBinaryContextListener { + /** + * @param typeId Type id. + * @param type Type. + */ + void onAfterMetadataRequest(int typeId, BinaryType type); + + /** + * @param typeId Type id. + * @param metadata Metadata. + */ + void onBeforeMetadataUpdate(int typeId, BinaryMetadata metadata); + } + + /** + * @param lsnr Listener. + */ + public void addListener(TestBinaryContextListener lsnr) { + if (listeners == null) + listeners = new ArrayList<>(); + + if (!listeners.contains(lsnr)) + listeners.add(lsnr); + } + + /** */ + public void clearAllListener() { + if (listeners != null) + listeners.clear(); + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java index 048abf62953ee..55462ff1edf81 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java @@ -423,6 +423,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscovery /** */ private IgniteDiscoverySpiInternalListener internalLsnr; + /** For test purposes. */ + private boolean skipAddrsRandomization = false; + /** * Gets current SPI state. * @@ -1881,7 +1884,7 @@ protected Collection resolvedAddresses() throws IgniteSpiExce } } - if (!res.isEmpty()) + if (!res.isEmpty() && !skipAddrsRandomization) Collections.shuffle(res); return res; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java index 84fd91656f9c2..41c588268a8f2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java @@ -100,6 +100,8 @@ public void testPreloadPartition() throws Exception { grid0 = startGrid(0); + grid0.cluster().active(true); + preloadPartition(grid0, DEFAULT_CACHE_NAME, PARTITION); Iterator> it = grid0.cache(DEFAULT_CACHE_NAME).iterator(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java index ccf4c8aa88ee6..ae75caa1bc74c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java @@ -819,7 +819,12 @@ private void testEnlistMany(boolean write) throws Exception { tx.commit(); } catch (Throwable t) { - assertTrue(X.hasCause(t, TransactionTimeoutException.class)); + boolean timedOut = X.hasCause(t, TransactionTimeoutException.class); + + if (!timedOut) + log.error("Got unexpected exception", t); + + assertTrue(timedOut); } assertEquals(0, client.cache(CACHE_NAME).size()); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java new file mode 100644 index 0000000000000..fed1d7f4ba1e4 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.Socket; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryType; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.QueryIndex; +import org.apache.ignite.cache.QueryIndexType; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.EventType; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.binary.BinaryMetadata; +import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper; +import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; +import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; +import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiClosure; +import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessage; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests scenario for too early metadata update completion in case of multiple concurrent updates for the same schema. + *

+ * Scenario is the following: + * + *

    + *
  • Start 4 nodes, connect client to node 2 in topology order (starting from 1).
  • + *
  • Start two concurrent transactions from client node producing same schema update.
  • + *
  • Delay second update until first update will return to client with stamped propose message and writes new + * schema to local metadata cache
  • + *
  • Unblock second update. It should correctly wait until the metadata is applied on all + * nodes or tx will fail on commit.
  • + *
+ */ +public class BinaryMetadataConcurrentUpdateWithIndexesTest extends GridCommonAbstractTest { + /** */ + private static final int FIELDS = 2; + + /** */ + private static final int MB = 1024 * 1024; + + /** */ + private static final TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + cfg.setIncludeEventTypes(EventType.EVTS_DISCOVERY); + + BlockTcpDiscoverySpi spi = new BlockTcpDiscoverySpi(); + + Field rndAddrsField = U.findField(BlockTcpDiscoverySpi.class, "skipAddrsRandomization"); + + assertNotNull(rndAddrsField); + + rndAddrsField.set(spi, true); + + cfg.setDiscoverySpi(spi.setIpFinder(ipFinder)); + + cfg.setClientMode(igniteInstanceName.startsWith("client")); + + QueryEntity qryEntity = new QueryEntity("java.lang.Integer", "Value"); + + LinkedHashMap fields = new LinkedHashMap<>(); + + Collection indexes = new ArrayList<>(FIELDS); + + for (int i = 0; i < FIELDS; i++) { + String name = "s" + i; + + fields.put(name, "java.lang.String"); + + indexes.add(new QueryIndex(name, QueryIndexType.SORTED)); + } + + qryEntity.setFields(fields); + + qryEntity.setIndexes(indexes); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration(). + setDefaultDataRegionConfiguration(new DataRegionConfiguration().setMaxSize(50 * MB))); + + cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME). + setBackups(0). + setQueryEntities(Collections.singleton(qryEntity)). + setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL). + setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC). + setCacheMode(CacheMode.PARTITIONED)); + + return cfg; + } + + /** Flag to start syncing metadata requests. Should skip on exchange. */ + private volatile boolean syncMeta; + + /** Metadata init latch. Both threads must request initial metadata. */ + private CountDownLatch initMetaReq = new CountDownLatch(2); + + /** Thread local flag for need of waiting local metadata update. */ + private ThreadLocal delayMetadataUpdateThreadLoc = new ThreadLocal<>(); + + /** Latch for waiting local metadata update. */ + public static final CountDownLatch localMetaUpdatedLatch = new CountDownLatch(1); + + /** */ + public void testMissingSchemaUpdate() throws Exception { + // Start order is important. + Ignite node0 = startGrid("node0"); + + Ignite node1 = startGrid("node1"); + + IgniteEx client0 = startGrid("client0"); + + CacheObjectBinaryProcessorImpl.TestBinaryContext clientCtx = + (CacheObjectBinaryProcessorImpl.TestBinaryContext)((CacheObjectBinaryProcessorImpl)client0.context(). + cacheObjects()).binaryContext(); + + clientCtx.addListener(new CacheObjectBinaryProcessorImpl.TestBinaryContext.TestBinaryContextListener() { + @Override public void onAfterMetadataRequest(int typeId, BinaryType type) { + if (syncMeta) { + try { + initMetaReq.countDown(); + + initMetaReq.await(); + } + catch (Exception e) { + throw new BinaryObjectException(e); + } + } + } + + @Override public void onBeforeMetadataUpdate(int typeId, BinaryMetadata metadata) { + // Delay one of updates until schema is locally updated on propose message. + if (delayMetadataUpdateThreadLoc.get() != null) + await(localMetaUpdatedLatch, 5000); + } + }); + + Ignite node2 = startGrid("node2"); + + Ignite node3 = startGrid("node3"); + + startGrid("node4"); + + node0.cluster().active(true); + + awaitPartitionMapExchange(); + + syncMeta = true; + + CountDownLatch clientProposeMsgBlockedLatch = new CountDownLatch(1); + + AtomicBoolean clientWait = new AtomicBoolean(); + final Object clientMux = new Object(); + + AtomicBoolean srvWait = new AtomicBoolean(); + final Object srvMux = new Object(); + + ((BlockTcpDiscoverySpi)node1.configuration().getDiscoverySpi()).setClosure((snd, msg) -> { + if (msg instanceof MetadataUpdateProposedMessage) { + if (Thread.currentThread().getName().contains("client")) { + log.info("Block custom message to client0: [locNode=" + snd + ", msg=" + msg + ']'); + + clientProposeMsgBlockedLatch.countDown(); + + // Message to client + synchronized (clientMux) { + while (!clientWait.get()) + try { + clientMux.wait(); + } + catch (InterruptedException e) { + fail(); + } + } + } + } + + return null; + }); + + ((BlockTcpDiscoverySpi)node2.configuration().getDiscoverySpi()).setClosure((snd, msg) -> { + if (msg instanceof MetadataUpdateProposedMessage) { + MetadataUpdateProposedMessage msg0 = (MetadataUpdateProposedMessage)msg; + + int pendingVer = U.field(msg0, "pendingVer"); + + // Should not block propose messages until they reach coordinator. + if (pendingVer == 0) + return null; + + log.info("Block custom message to next server: [locNode=" + snd + ", msg=" + msg + ']'); + + // Message to client + synchronized (srvMux) { + while (!srvWait.get()) + try { + srvMux.wait(); + } + catch (InterruptedException e) { + fail(); + } + } + } + + return null; + }); + + Integer key = primaryKey(node3.cache(DEFAULT_CACHE_NAME)); + + IgniteInternalFuture fut0 = runAsync(() -> { + try (Transaction tx = client0.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + client0.cache(DEFAULT_CACHE_NAME).put(key, build(client0, "val", 0)); + + tx.commit(); + } + catch (Throwable t) { + log.error("err", t); + } + + }); + + // Implements test logic. + IgniteInternalFuture fut1 = runAsync(() -> { + // Wait for initial metadata received. It should be initial version: pending=0, accepted=0 + await(initMetaReq, 5000); + + // Wait for blocking proposal message to client node. + await(clientProposeMsgBlockedLatch, 5000); + + // Unblock proposal message to client. + clientWait.set(true); + + synchronized (clientMux) { + clientMux.notify(); + } + + // Give some time to apply update. + doSleep(3000); + + // Unblock second metadata update. + localMetaUpdatedLatch.countDown(); + + // Give some time for tx to complete (success or fail). fut2 will throw an error if tx has failed on commit. + doSleep(3000); + + // Unblock metadata message and allow for correct version acceptance. + srvWait.set(true); + + synchronized (srvMux) { + srvMux.notify(); + } + }); + + IgniteInternalFuture fut2 = runAsync(() -> { + delayMetadataUpdateThreadLoc.set(true); + + try (Transaction tx = client0.transactions(). + txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) { + client0.cache(DEFAULT_CACHE_NAME).put(key, build(client0, "val", 0)); + + tx.commit(); + } + }); + + fut0.get(); + fut1.get(); + fut2.get(); + } + + /** + * @param latch Latch. + * @param timeout Timeout. + */ + private void await(CountDownLatch latch, long timeout) { + try { + latch.await(5000, MILLISECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + long cnt = initMetaReq.getCount(); + + if (cnt != 0) + throw new RuntimeException("Invalid latch count after wait: " + cnt); + } + + /** + * @param ignite Ignite. + * @param prefix Value prefix. + * @param fields Fields. + */ + protected BinaryObject build(Ignite ignite, String prefix, int... fields) { + BinaryObjectBuilder builder = ignite.binary().builder("Value"); + + for (int field : fields) { + assertTrue(field < FIELDS); + + builder.setField("i" + field, field); + builder.setField("s" + field, prefix + field); + } + + return builder.build(); + } + + /** + * Discovery SPI which can simulate network split. + */ + protected class BlockTcpDiscoverySpi extends TcpDiscoverySpi { + /** Closure. */ + private volatile IgniteBiClosure clo; + + /** + * @param clo Closure. + */ + public void setClosure(IgniteBiClosure clo) { + this.clo = clo; + } + + /** + * @param addr Address. + * @param msg Message. + */ + private synchronized void apply(ClusterNode addr, TcpDiscoveryAbstractMessage msg) { + if (!(msg instanceof TcpDiscoveryCustomEventMessage)) + return; + + TcpDiscoveryCustomEventMessage cm = (TcpDiscoveryCustomEventMessage)msg; + + DiscoveryCustomMessage delegate; + + try { + DiscoverySpiCustomMessage custMsg = cm.message(marshaller(), U.resolveClassLoader(ignite().configuration())); + + assertNotNull(custMsg); + + delegate = ((CustomMessageWrapper)custMsg).delegate(); + + } + catch (Throwable throwable) { + throw new RuntimeException(throwable); + } + + if (clo != null) + clo.apply(addr, delegate); + } + + /** {@inheritDoc} */ + @Override protected void writeToSocket( + Socket sock, + TcpDiscoveryAbstractMessage msg, + byte[] data, + long timeout + ) throws IOException { + if (spiCtx != null) + apply(spiCtx.localNode(), msg); + + super.writeToSocket(sock, msg, data, timeout); + } + + /** {@inheritDoc} */ + @Override protected void writeToSocket(Socket sock, + OutputStream out, + TcpDiscoveryAbstractMessage msg, + long timeout) throws IOException, IgniteCheckedException { + if (spiCtx != null) + apply(spiCtx.localNode(), msg); + + super.writeToSocket(sock, out, msg, timeout); + } + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + CacheObjectBinaryProcessorImpl.useTestBinaryCtx = true; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + CacheObjectBinaryProcessorImpl.useTestBinaryCtx = false; + + stopAllGrids(); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java index b44ff2dbc3517..8a60c7d72ac70 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.BinaryMetadataConcurrentUpdateWithIndexesTest; import org.apache.ignite.internal.processors.cache.BinarySerializationQuerySelfTest; import org.apache.ignite.internal.processors.cache.BinarySerializationQueryWithReflectiveSerializerSelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanSelfTest; @@ -42,6 +43,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheBinaryObjectsScanWithEventsSelfTest.class); suite.addTestSuite(BigEntryQueryTest.class); + suite.addTestSuite(BinaryMetadataConcurrentUpdateWithIndexesTest.class); + //Should be adjusted. Not ready to be used with BinaryMarshaller. //suite.addTestSuite(GridCacheBinarySwapScanQuerySelfTest.class); From 9f9b9e45e72b04e13aa714cbc3906703e9e0455f Mon Sep 17 00:00:00 2001 From: Evgeny Stanilovskiy Date: Wed, 17 Oct 2018 16:35:32 +0300 Subject: [PATCH 018/403] IGNITE-9875 Optimized GridDhtPartitionsStateValidator - Fixes #4983. Signed-off-by: Alexey Goncharuk --- modules/benchmarks/pom.xml | 12 +- ...dDhtPartitionsStateValidatorBenchmark.java | 168 ++++++++++++++++++ .../CachePartitionFullCountersMap.java | 21 --- .../GridDhtPartitionsSingleMessage.java | 29 +++ .../GridDhtPartitionsStateValidator.java | 84 +++++---- 5 files changed, 257 insertions(+), 57 deletions(-) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml index 1ea984c6c544e..06e0e505cd2ae 100644 --- a/modules/benchmarks/pom.xml +++ b/modules/benchmarks/pom.xml @@ -62,6 +62,16 @@ ${jmh.version} provided + + org.mockito + mockito-all + ${mockito.version} + + + com.google.guava + guava + ${guava.version} + @@ -131,4 +141,4 @@ - \ No newline at end of file + diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java new file mode 100644 index 0000000000000..151606dab21ba --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java @@ -0,0 +1,168 @@ +package org.apache.ignite.internal.benchmarks.jmh.misc; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.ignite.internal.benchmarks.jmh.JmhAbstractBenchmark; +import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator; +import org.apache.ignite.internal.util.typedef.T2; +import org.jetbrains.annotations.Nullable; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.IntStream; + +import static org.openjdk.jmh.annotations.Scope.Thread; + +/** */ +@State(Scope.Benchmark) +public class GridDhtPartitionsStateValidatorBenchmark extends JmhAbstractBenchmark { + /** */ + @State(Thread) + public static class Context { + /** */ + private final UUID localNodeId = UUID.randomUUID(); + + /** */ + private GridCacheSharedContext cctxMock; + + /** */ + private GridDhtPartitionTopology topologyMock; + + /** */ + private GridDhtPartitionsStateValidator validator; + + /** */ + private Map messages = new HashMap<>(); + + /** */ + private UUID ignoreNode = UUID.randomUUID(); + + /** */ + private static final int NODES = 3; + + /** */ + private static final int PARTS = 100; + + /** + * @return Partition mock with specified {@code id}, {@code updateCounter} and {@code size}. + */ + private GridDhtLocalPartition partitionMock(int id, long updateCounter, long size) { + GridDhtLocalPartition partitionMock = Mockito.mock(GridDhtLocalPartition.class); + Mockito.when(partitionMock.id()).thenReturn(id); + Mockito.when(partitionMock.updateCounter()).thenReturn(updateCounter); + Mockito.when(partitionMock.fullSize()).thenReturn(size); + Mockito.when(partitionMock.state()).thenReturn(GridDhtPartitionState.OWNING); + return partitionMock; + } + + /** + * @param countersMap Update counters map. + * @param sizesMap Sizes map. + * @return Message with specified {@code countersMap} and {@code sizeMap}. + */ + private GridDhtPartitionsSingleMessage from(@Nullable Map> countersMap, @Nullable Map sizesMap) { + GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage(); + if (countersMap != null) + msg.addPartitionUpdateCounters(0, countersMap); + if (sizesMap != null) + msg.addPartitionSizes(0, sizesMap); + return msg; + } + + /** */ + @Setup + public void setup() { + // Prepare mocks. + cctxMock = Mockito.mock(GridCacheSharedContext.class); + Mockito.when(cctxMock.localNodeId()).thenReturn(localNodeId); + + topologyMock = Mockito.mock(GridDhtPartitionTopology.class); + Mockito.when(topologyMock.partitionState(Matchers.any(), Matchers.anyInt())).thenReturn(GridDhtPartitionState.OWNING); + Mockito.when(topologyMock.groupId()).thenReturn(0); + + Mockito.when(topologyMock.partitions()).thenReturn(PARTS); + + List localPartitions = Lists.newArrayList(); + + Map> updateCountersMap = new HashMap<>(); + + Map cacheSizesMap = new HashMap<>(); + + IntStream.range(0, PARTS).forEach(k -> { localPartitions.add(partitionMock(k, k + 1, k + 1)); + long us = k > 20 && k <= 30 ? 0 :k + 2L; + updateCountersMap.put(k, new T2<>(k + 2L, us)); + cacheSizesMap.put(k, us); }); + + Mockito.when(topologyMock.localPartitions()).thenReturn(localPartitions); + Mockito.when(topologyMock.currentLocalPartitions()).thenReturn(localPartitions); + + // Form single messages map. + Map messages = new HashMap<>(); + + for (int n = 0; n < NODES; ++n) { + UUID remoteNode = UUID.randomUUID(); + + messages.put(remoteNode, from(updateCountersMap, cacheSizesMap)); + } + + messages.put(ignoreNode, from(updateCountersMap, cacheSizesMap)); + + validator = new GridDhtPartitionsStateValidator(cctxMock); + } + } + + /** */ + @Benchmark + public void testValidatePartitionsUpdateCounters(Context context) { + context.validator.validatePartitionsUpdateCounters(context.topologyMock, + context.messages, Sets.newHashSet(context.ignoreNode)); + } + + /** */ + @Benchmark + public void testValidatePartitionsSizes(Context context) { + context.validator.validatePartitionsSizes(context.topologyMock, context + .messages, Sets.newHashSet(context.ignoreNode)); + } + + /** + * Run benchmarks. + * + * @param args Arguments. + * @throws Exception If failed. + */ + public static void main(String[] args) throws Exception { + run(1); + } + + /** + * Run benchmark. + * + * @param threads Amount of threads. + * @throws Exception If failed. + */ + private static void run(int threads) throws Exception { + JmhIdeBenchmarkRunner.create() + .forks(1) + .threads(threads) + .warmupIterations(5) + .measurementIterations(10) + .benchmarks(GridDhtPartitionsStateValidatorBenchmark.class.getSimpleName()) + .jvmArguments("-XX:+UseG1GC", "-Xms4g", "-Xmx4g") + .run(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java index 2d5eec3878cf5..008c2766e4a47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java @@ -93,27 +93,6 @@ public void updateCounter(int p, long updCntr) { updCntrs[p] = updCntr; } - /** - * Creates submap for provided partition IDs. - * - * @param parts Partition IDs. - * @return Partial counters map. - */ - public CachePartitionPartialCountersMap subMap(Set parts) { - CachePartitionPartialCountersMap res = new CachePartitionPartialCountersMap(parts.size()); - - for (int p = 0; p < updCntrs.length; p++) { - if (!parts.contains(p)) - continue; - - res.add(p, initialUpdCntrs[p], updCntrs[p]); - } - - assert res.size() == parts.size(); - - return res; - } - /** * Clears full counters map. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index 088fb31d7f715..7dd34f80e00bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -245,6 +245,35 @@ public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int p return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt); } + /** + * @param grpId Cache group ID. + * @param partsCnt Total cache partitions. + * @return Partition update counters. + */ + @SuppressWarnings("unchecked") + public CachePartitionPartialCountersMap partitionUpdateCountersUnsorted(int grpId, int partsCnt) { + Object res = partCntrs == null ? null : partCntrs.get(grpId); + + if (res == null) + return CachePartitionPartialCountersMap.EMPTY; + + if (res instanceof CachePartitionPartialCountersMap) + return (CachePartitionPartialCountersMap)res; + + assert res instanceof Map : res; + + Map> map = (Map>)res; + + CachePartitionPartialCountersMap partCounersMap = new CachePartitionPartialCountersMap(partsCnt); + + for (Map.Entry> e : map.entrySet()) + partCounersMap.add(e.getKey(), e.getValue().get1(), e.getValue().get2()); + + partCounersMap.trim(); + + return partCounersMap; + } + /** * Adds partition sizes map for specified {@code grpId} to the current message. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java index 544d4531524d1..63fe926acc60f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.topology; +import java.util.AbstractMap; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -33,7 +34,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.lang.IgniteProductVersion; import org.jetbrains.annotations.Nullable; @@ -78,14 +78,16 @@ public void validatePartitionCountersAndSizes( final Set ignoringNodes = new HashSet<>(); // Ignore just joined nodes. - for (DiscoveryEvent evt : fut.events().events()) + for (DiscoveryEvent evt : fut.events().events()) { if (evt.type() == EVT_NODE_JOINED) ignoringNodes.add(evt.eventNode().id()); + } AffinityTopologyVersion topVer = fut.context().events().topologyVersion(); // Validate update counters. Map> result = validatePartitionsUpdateCounters(top, messages, ignoringNodes); + if (!result.isEmpty()) throw new IgniteCheckedException("Partitions update counters are inconsistent for " + fold(topVer, result)); @@ -110,13 +112,16 @@ public void validatePartitionCountersAndSizes( * * @param top Topology to validate. * @param nodeId Node which sent single message. - * @param singleMsg Single message. + * @param countersMap Counters map. + * @param sizesMap Sizes map. * @return Set of partition ids should be excluded from validation. */ - @Nullable private Set shouldIgnore(GridDhtPartitionTopology top, UUID nodeId, GridDhtPartitionsSingleMessage singleMsg) { - CachePartitionPartialCountersMap countersMap = singleMsg.partitionUpdateCounters(top.groupId(), top.partitions()); - Map sizesMap = singleMsg.partitionSizes(top.groupId()); - + @Nullable private Set shouldIgnore( + GridDhtPartitionTopology top, + UUID nodeId, + CachePartitionPartialCountersMap countersMap, + Map sizesMap + ) { Set ignore = null; for (int i = 0; i < countersMap.size(); i++) { @@ -155,14 +160,14 @@ public void validatePartitionCountersAndSizes( * @return Invalid partitions map with following structure: (partId, (nodeId, updateCounter)). * If map is empty validation is successful. */ - public Map> validatePartitionsUpdateCounters( - GridDhtPartitionTopology top, - Map messages, - Set ignoringNodes - ) { + public Map> validatePartitionsUpdateCounters( + GridDhtPartitionTopology top, + Map messages, + Set ignoringNodes + ) { Map> invalidPartitions = new HashMap<>(); - Map> updateCountersAndNodesByPartitions = new HashMap<>(); + Map> updateCountersAndNodesByPartitions = new HashMap<>(); // Populate counters statistics from local node partitions. for (GridDhtLocalPartition part : top.currentLocalPartitions()) { @@ -172,7 +177,7 @@ public Map> validatePartitionsUpdateCounters( if (part.updateCounter() == 0 && part.fullSize() == 0) continue; - updateCountersAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.updateCounter())); + updateCountersAndNodesByPartitions.put(part.id(), new AbstractMap.SimpleEntry<>(cctx.localNodeId(), part.updateCounter())); } int partitions = top.partitions(); @@ -183,9 +188,13 @@ public Map> validatePartitionsUpdateCounters( if (ignoringNodes.contains(nodeId)) continue; - CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions); + final GridDhtPartitionsSingleMessage message = e.getValue(); - Set ignorePartitions = shouldIgnore(top, nodeId, e.getValue()); + CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions); + + Map sizesMap = message.partitionSizes(top.groupId()); + + Set ignorePartitions = shouldIgnore(top, nodeId, countersMap, sizesMap); for (int i = 0; i < countersMap.size(); i++) { int p = countersMap.partitionAt(i); @@ -211,14 +220,14 @@ public Map> validatePartitionsUpdateCounters( * @return Invalid partitions map with following structure: (partId, (nodeId, cacheSize)). * If map is empty validation is successful. */ - public Map> validatePartitionsSizes( - GridDhtPartitionTopology top, - Map messages, - Set ignoringNodes - ) { + public Map> validatePartitionsSizes( + GridDhtPartitionTopology top, + Map messages, + Set ignoringNodes + ) { Map> invalidPartitions = new HashMap<>(); - Map> sizesAndNodesByPartitions = new HashMap<>(); + Map> sizesAndNodesByPartitions = new HashMap<>(); // Populate sizes statistics from local node partitions. for (GridDhtLocalPartition part : top.currentLocalPartitions()) { @@ -228,7 +237,7 @@ public Map> validatePartitionsSizes( if (part.updateCounter() == 0 && part.fullSize() == 0) continue; - sizesAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.fullSize())); + sizesAndNodesByPartitions.put(part.id(), new AbstractMap.SimpleEntry<>(cctx.localNodeId(), part.fullSize())); } int partitions = top.partitions(); @@ -239,10 +248,13 @@ public Map> validatePartitionsSizes( if (ignoringNodes.contains(nodeId)) continue; - CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions); - Map sizesMap = e.getValue().partitionSizes(top.groupId()); + final GridDhtPartitionsSingleMessage message = e.getValue(); + + CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions); + + Map sizesMap = message.partitionSizes(top.groupId()); - Set ignorePartitions = shouldIgnore(top, nodeId, e.getValue()); + Set ignorePartitions = shouldIgnore(top, nodeId, countersMap, sizesMap); for (int i = 0; i < countersMap.size(); i++) { int p = countersMap.partitionAt(i); @@ -269,20 +281,22 @@ public Map> validatePartitionsSizes( * @param node Node id. * @param counter Counter value reported by {@code node}. */ - private void process(Map> invalidPartitions, - Map> countersAndNodes, - int part, - UUID node, - long counter) { - T2 existingData = countersAndNodes.get(part); + private void process( + Map> invalidPartitions, + Map> countersAndNodes, + int part, + UUID node, + long counter + ) { + AbstractMap.Entry existingData = countersAndNodes.get(part); if (existingData == null) - countersAndNodes.put(part, new T2<>(node, counter)); + countersAndNodes.put(part, new AbstractMap.SimpleEntry<>(node, counter)); - if (existingData != null && counter != existingData.get2()) { + if (existingData != null && counter != existingData.getValue()) { if (!invalidPartitions.containsKey(part)) { Map map = new HashMap<>(); - map.put(existingData.get1(), existingData.get2()); + map.put(existingData.getKey(), existingData.getValue()); invalidPartitions.put(part, map); } From 7a1cd6b0d93419418464601686f67001a025ea62 Mon Sep 17 00:00:00 2001 From: pereslegin-pa Date: Wed, 17 Oct 2018 19:58:29 +0300 Subject: [PATCH 019/403] IGNITE-9756 Fixed partition eviction deduplication logic - Fixes #4936. Signed-off-by: Alexey Goncharuk --- .../dht/topology/PartitionsEvictManager.java | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index cd010fa89d32d..238c6cde05163 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -19,6 +19,7 @@ import java.util.Collection; import java.util.Comparator; +import java.util.HashSet; import java.util.Map; import java.util.Queue; import java.util.Set; @@ -31,7 +32,6 @@ import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; -import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; @@ -110,28 +110,28 @@ public void onCacheGroupStopped(CacheGroupContext grp){ * @param part Partition to evict. */ public void evictPartitionAsync(CacheGroupContext grp, GridDhtLocalPartition part) { - // Check node stop. - if (sharedEvictionContext.shouldStop()) - return; - GroupEvictionContext groupEvictionContext = evictionGroupsMap.computeIfAbsent( grp.groupId(), (k) -> new GroupEvictionContext(grp)); - PartitionEvictionTask evictionTask = groupEvictionContext.createEvictPartitionTask(part); - - if (evictionTask == null) + // Check node stop. + if (groupEvictionContext.shouldStop()) return; - if (log.isDebugEnabled()) - log.debug("Partition has been scheduled for eviction [grp=" + grp.cacheOrGroupName() - + ", p=" + part.id() + ", state=" + part.state() + "]"); - int bucket; synchronized (mux) { - bucket = evictionQueue.offer(evictionTask); + if (!groupEvictionContext.partIds.add(part.id())) + return; + + bucket = evictionQueue.offer(new PartitionEvictionTask(part, groupEvictionContext)); } + groupEvictionContext.totalTasks.incrementAndGet(); + + if (log.isDebugEnabled()) + log.debug("Partition has been scheduled for eviction [grp=" + grp.cacheOrGroupName() + + ", p=" + part.id() + ", state=" + part.state() + "]"); + scheduleNextPartitionEviction(bucket); } @@ -270,7 +270,7 @@ private class GroupEvictionContext implements EvictionContext { private final CacheGroupContext grp; /** Deduplicate set partition ids. */ - private final Set partIds = new GridConcurrentHashSet<>(); + private final Set partIds = new HashSet<>(); /** Future for currently running partition eviction task. */ private final Map> partsEvictFutures = new ConcurrentHashMap<>(); @@ -296,19 +296,6 @@ private GroupEvictionContext(CacheGroupContext grp) { return stop || sharedEvictionContext.shouldStop(); } - /** - * - * @param part Grid local partition. - */ - private PartitionEvictionTask createEvictPartitionTask(GridDhtLocalPartition part){ - if (shouldStop() || !partIds.add(part.id())) - return null; - - totalTasks.incrementAndGet(); - - return new PartitionEvictionTask(part, this); - } - /** * * @param task Partition eviction task. @@ -323,6 +310,8 @@ private synchronized void taskScheduled(PartitionEvictionTask task) { int partId = task.part.id(); + partIds.remove(partId); + partsEvictFutures.put(partId, fut); fut.listen(f -> { @@ -426,12 +415,14 @@ private PartitionEvictionTask( if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy()) part.destroy(); } - else // Re-offer partition if clear was unsuccessful due to partition reservation. - evictionQueue.offer(this); // Complete eviction future before schedule new to prevent deadlock with // simultaneous eviction stopping and scheduling new eviction. finishFut.onDone(); + + // Re-offer partition if clear was unsuccessful due to partition reservation. + if (!success) + evictPartitionAsync(groupEvictionCtx.grp, part); } catch (Throwable ex) { finishFut.onDone(ex); From b91ddd08e0c5b368a423d3e8a073b23afcef512f Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Wed, 17 Oct 2018 20:08:34 +0300 Subject: [PATCH 020/403] IGNITE-9868 Improved background full message sending - Fixes #4975. Signed-off-by: Alexey Goncharuk --- .../GridCachePartitionExchangeManager.java | 177 +++++++++++++----- .../GridDhtPartitionsExchangeFuture.java | 12 +- 2 files changed, 135 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index dbfc3e4be889a..d30b002f1a775 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -1028,11 +1028,13 @@ public void scheduleResendPartitions() { } /** - * Partition refresh callback. + * Partition refresh callback for selected cache groups. * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send, * for non coordinator - {@link GridDhtPartitionsSingleMessage SingleMessages} send + * + * @param grps Cache groups for partitions refresh. */ - public void refreshPartitions() { + public void refreshPartitions(@NotNull Collection grps) { // TODO https://issues.apache.org/jira/browse/IGNITE-6857 if (cctx.snapshot().snapshotOperationInProgress()) { scheduleResendPartitions(); @@ -1040,6 +1042,13 @@ public void refreshPartitions() { return; } + if (grps.isEmpty()) { + if (log.isDebugEnabled()) + log.debug("Skip partitions refresh, there are no cache groups for partition refresh."); + + return; + } + ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE); if (oldest == null) { @@ -1049,8 +1058,10 @@ public void refreshPartitions() { return; } - if (log.isDebugEnabled()) - log.debug("Refreshing partitions [oldest=" + oldest.id() + ", loc=" + cctx.localNodeId() + ']'); + if (log.isDebugEnabled()) { + log.debug("Refreshing partitions [oldest=" + oldest.id() + ", loc=" + cctx.localNodeId() + + ", cacheGroups= " + grps + ']'); + } // If this is the oldest node. if (oldest.id().equals(cctx.localNodeId())) { @@ -1068,50 +1079,66 @@ public void refreshPartitions() { // No need to send to nodes which did not finish their first exchange. AffinityTopologyVersion rmtTopVer = - lastFut != null ? (lastFut.isDone() ? lastFut.topologyVersion() : lastFut.initialVersion()) : AffinityTopologyVersion.NONE; + lastFut != null ? + (lastFut.isDone() ? lastFut.topologyVersion() : lastFut.initialVersion()) + : AffinityTopologyVersion.NONE; Collection rmts = cctx.discovery().remoteAliveNodesWithCaches(rmtTopVer); if (log.isDebugEnabled()) log.debug("Refreshing partitions from oldest node: " + cctx.localNodeId()); - sendAllPartitions(rmts, rmtTopVer); + sendAllPartitions(rmts, rmtTopVer, grps); } else { if (log.isDebugEnabled()) log.debug("Refreshing local partitions from non-oldest node: " + cctx.localNodeId()); - sendLocalPartitions(oldest, null); + sendLocalPartitions(oldest, null, grps); } } + /** + * Partition refresh callback. + * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send, + * for non coordinator - {@link GridDhtPartitionsSingleMessage SingleMessages} send + */ + public void refreshPartitions() { refreshPartitions(cctx.cache().cacheGroups()); } + /** * @param nodes Nodes. * @param msgTopVer Topology version. Will be added to full message. + * @param grps Selected cache groups. */ private void sendAllPartitions( Collection nodes, - AffinityTopologyVersion msgTopVer + AffinityTopologyVersion msgTopVer, + Collection grps ) { long time = System.currentTimeMillis(); - GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null); + GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null, grps); m.topologyVersion(msgTopVer); if (log.isInfoEnabled()) { long latency = System.currentTimeMillis() - time; - if (latency > 100 || log.isDebugEnabled()) - log.info("Full Message creating for " + msgTopVer + " performed in " + latency + " ms."); + if (latency > 50 || log.isDebugEnabled()) { + log.info("Finished full message creation [msgTopVer=" + msgTopVer + ", groups=" + grps + + ", latency=" + latency + "ms]"); + } } if (log.isTraceEnabled()) - log.trace("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", msg=" + m + ']'); + log.trace("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", cacheGroups=" + grps + + ", msg=" + m + ']'); time = System.currentTimeMillis(); + Collection failedNodes = U.newHashSet(nodes.size()); + for (ClusterNode node : nodes) { try { assert !node.equals(cctx.localNode()); @@ -1119,22 +1146,34 @@ private void sendAllPartitions( cctx.io().sendNoRetry(node, m, SYSTEM_POOL); } catch (ClusterTopologyCheckedException ignore) { - if (log.isDebugEnabled()) - log.debug("Failed to send partition update to node because it left grid (will ignore) [node=" + - node.id() + ", msg=" + m + ']'); + if (log.isDebugEnabled()) { + log.debug("Failed to send partition update to node because it left grid (will ignore) " + + "[node=" + node.id() + ", msg=" + m + ']'); + } } catch (IgniteCheckedException e) { - U.warn(log, "Failed to send partitions full message [node=" + node + ", err=" + e + ']'); + failedNodes.add(node); + + U.warn(log, "Failed to send partitions full message [node=" + node + ", err=" + e + ']', e); } } - if (log.isInfoEnabled()) - log.info("Sending Full Message for " + msgTopVer + " performed in " + (System.currentTimeMillis() - time) + " ms."); + if (log.isInfoEnabled()) { + long latency = System.currentTimeMillis() - time; + + if (latency > 50 || log.isDebugEnabled()) { + log.info("Finished sending full message [msgTopVer=" + msgTopVer + ", groups=" + grps + + (failedNodes.isEmpty() ? "" : (", skipped=" + U.nodeIds(failedNodes))) + + ", latency=" + latency + "ms]"); + } + } } /** + * Creates partitions full message for all cache groups. + * * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/ - * finishUnmarshall methods are called). + * finishUnmarshall methods are called). * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}. * @param exchId Non-null exchange ID if message is created for exchange. * @param lastVer Last version. @@ -1150,18 +1189,43 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers, @Nullable IgniteDhtPartitionsToReloadMap partsToReload ) { - final GridDhtPartitionsFullMessage m = new GridDhtPartitionsFullMessage(exchId, - lastVer, - exchId != null ? exchId.topologyVersion() : AffinityTopologyVersion.NONE, - partHistSuppliers, - partsToReload - ); + Collection grps = cctx.cache().cacheGroups(); + + return createPartitionsFullMessage(compress, newCntrMap, exchId, lastVer, partHistSuppliers, partsToReload, grps); + } + + /** + * Creates partitions full message for selected cache groups. + * + * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/ + * finishUnmarshall methods are called). + * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}. + * @param exchId Non-null exchange ID if message is created for exchange. + * @param lastVer Last version. + * @param partHistSuppliers Partition history suppliers map. + * @param partsToReload Partitions to reload map. + * @param grps Selected cache groups. + * @return Message. + */ + public GridDhtPartitionsFullMessage createPartitionsFullMessage( + boolean compress, + boolean newCntrMap, + @Nullable final GridDhtPartitionExchangeId exchId, + @Nullable GridCacheVersion lastVer, + @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers, + @Nullable IgniteDhtPartitionsToReloadMap partsToReload, + Collection grps + ) { + AffinityTopologyVersion ver = exchId != null ? exchId.topologyVersion() : AffinityTopologyVersion.NONE; + + final GridDhtPartitionsFullMessage m = + new GridDhtPartitionsFullMessage(exchId, lastVer, ver, partHistSuppliers, partsToReload); m.compress(compress); final Map> dupData = new HashMap<>(); - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { + for (CacheGroupContext grp : grps) { if (!grp.isLocal()) { if (exchId != null) { AffinityTopologyVersion startTopVer = grp.localStartVersion(); @@ -1174,14 +1238,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( GridDhtPartitionFullMap locMap = grp.topology().partitionMap(true); - if (locMap != null) { - addFullPartitionsMap(m, - dupData, - compress, - grp.groupId(), - locMap, - affCache.similarAffinityKey()); - } + if (locMap != null) + addFullPartitionsMap(m, dupData, compress, grp.groupId(), locMap, affCache.similarAffinityKey()); m.addPartitionSizes(grp.groupId(), grp.topology().globalPartSizes()); @@ -1202,14 +1260,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) { GridDhtPartitionFullMap map = top.partitionMap(true); - if (map != null) { - addFullPartitionsMap(m, - dupData, - compress, - top.groupId(), - map, - top.similarAffinityKey()); - } + if (map != null) + addFullPartitionsMap(m, dupData, compress, top.groupId(), map, top.similarAffinityKey()); if (exchId != null) { CachePartitionFullCountersMap cntrsMap = top.fullUpdateCounters(); @@ -1269,13 +1321,15 @@ private void addFullPartitionsMap(GridDhtPartitionsFullMessage m, /** * @param node Destination cluster node. * @param id Exchange ID. + * @param grps Cache groups for send partitions. */ - private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExchangeId id) { - GridDhtPartitionsSingleMessage m = createPartitionsSingleMessage(id, - cctx.kernalContext().clientNode(), - false, - false, - null); + private void sendLocalPartitions( + ClusterNode node, + @Nullable GridDhtPartitionExchangeId id, + @NotNull Collection grps + ) { + GridDhtPartitionsSingleMessage m = + createPartitionsSingleMessage(id, cctx.kernalContext().clientNode(), false, false, null, grps); if (log.isTraceEnabled()) log.trace("Sending local partitions [nodeId=" + node.id() + ", msg=" + m + ']'); @@ -1294,6 +1348,8 @@ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExc } /** + * Creates partitions single message for all cache groups. + * * @param exchangeId Exchange ID. * @param clientOnlyExchange Client exchange flag. * @param sndCounters {@code True} if need send partition update counters. @@ -1306,6 +1362,29 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( boolean sndCounters, boolean newCntrMap, ExchangeActions exchActions + ) { + Collection grps = cctx.cache().cacheGroups(); + + return createPartitionsSingleMessage(exchangeId, clientOnlyExchange, sndCounters, newCntrMap, exchActions, grps); + } + + /** + * Creates partitions single message for selected cache groups. + * + * @param exchangeId Exchange ID. + * @param clientOnlyExchange Client exchange flag. + * @param sndCounters {@code True} if need send partition update counters. + * @param newCntrMap {@code True} if possible to use {@link CachePartitionPartialCountersMap}. + * @param grps Selected cache groups. + * @return Message. + */ + public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( + @Nullable GridDhtPartitionExchangeId exchangeId, + boolean clientOnlyExchange, + boolean sndCounters, + boolean newCntrMap, + ExchangeActions exchActions, + Collection grps ) { GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(exchangeId, clientOnlyExchange, @@ -1314,7 +1393,7 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( Map> dupData = new HashMap<>(); - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { + for (CacheGroupContext grp : grps) { if (!grp.isLocal() && (exchActions == null || !exchActions.cacheGroupStopping(grp.groupId()))) { GridDhtPartitionMap locMap = grp.topology().localPartitionMap(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index b2362e299f806..16d7570fba35f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -1868,22 +1868,24 @@ public void finishMerged() { if (centralizedAff || forceAffReassignment) { assert !exchCtx.mergeExchanges(); + Collection grpToRefresh = U.newHashSet(cctx.cache().cacheGroups().size()); + for (CacheGroupContext grp : cctx.cache().cacheGroups()) { if (grp.isLocal()) continue; - boolean needRefresh = false; - try { - needRefresh = grp.topology().initPartitionsWhenAffinityReady(res, this); + if (grp.topology().initPartitionsWhenAffinityReady(res, this)) + grpToRefresh.add(grp); } catch (IgniteInterruptedCheckedException e) { U.error(log, "Failed to initialize partitions.", e); } - if (needRefresh) - cctx.exchange().refreshPartitions(); } + + if (!grpToRefresh.isEmpty()) + cctx.exchange().refreshPartitions(grpToRefresh); } for (GridCacheContext cacheCtx : cctx.cacheContexts()) { From 12c3287730a058f5d4630bc43cff4f14c308cbd8 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Wed, 17 Oct 2018 20:30:27 +0300 Subject: [PATCH 021/403] IGNITE-9756 Fixed compilation --- .../cache/distributed/dht/topology/PartitionsEvictManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index 238c6cde05163..27b11c242bd20 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -422,7 +422,7 @@ private PartitionEvictionTask( // Re-offer partition if clear was unsuccessful due to partition reservation. if (!success) - evictPartitionAsync(groupEvictionCtx.grp, part); + evictPartitionAsync(groupEvictionContext.grp, part); } catch (Throwable ex) { finishFut.onDone(ex); From 926b0314276d5b2fca8a893ce108bddc31a6d0d2 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Thu, 18 Oct 2018 14:33:28 +0300 Subject: [PATCH 022/403] IGNITE-9854 Correct remove from dirtyPages and segCheckpointPages - Fixes #4988. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit 829dc1f240c07731a1ee98ae18c80ea6074dc6c4) --- .../persistence/pagemem/PageMemoryImpl.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 55c5fb53b8886..48e33c08b54e3 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -851,11 +851,15 @@ private long refreshOutdatedPage(Segment seg, int grpId, long pageId, boolean rm if (rmv) seg.loadedPages.remove(grpId, PageIdUtils.effectivePageId(pageId)); - if (seg.segCheckpointPages != null) - seg.segCheckpointPages.remove(new FullPageId(pageId, grpId)); + Collection cpPages = seg.segCheckpointPages; - if (seg.dirtyPages != null) - seg.dirtyPages.remove(new FullPageId(pageId, grpId)); + if (cpPages != null) + cpPages.remove(new FullPageId(pageId, grpId)); + + Collection dirtyPages = seg.dirtyPages; + + if (dirtyPages != null) + dirtyPages.remove(new FullPageId(pageId, grpId)); return relPtr; } @@ -1875,7 +1879,7 @@ private class Segment extends ReentrantReadWriteLock { private static final int ACQUIRED_PAGES_PADDING = 4; /** Page ID to relative pointer map. */ - private LoadedPagesMap loadedPages; + private final LoadedPagesMap loadedPages; /** Pointer to acquired pages integer counter. */ private long acquiredPagesPtr; @@ -1887,7 +1891,7 @@ private class Segment extends ReentrantReadWriteLock { private long memPerTbl; /** Pages marked as dirty since the last checkpoint. */ - private Collection dirtyPages = new GridConcurrentHashSet<>(); + private volatile Collection dirtyPages = new GridConcurrentHashSet<>(); /** */ private volatile Collection segCheckpointPages; From 22d7560012daaa1e4bb259bc2b0ae54f8be9de23 Mon Sep 17 00:00:00 2001 From: Evgeny Stanilovskiy Date: Wed, 17 Oct 2018 17:56:08 +0300 Subject: [PATCH 023/403] IGNITE-9272: j.u.zip.CRC32algo instead of PureJavaCrc32 - Fixes #4619. Signed-off-by: Nikolay Izhikov (cherry picked from commit 091ace90e56bac1f7bc76886a5dccff796cbe34e) --- .../benchmarks/jmh/algo/BenchmarkCRC.java | 95 ++++++++++++++++ .../GridCacheDatabaseSharedManager.java | 4 +- .../persistence/file/EncryptedFileIO.java | 7 +- .../cache/persistence/file/FilePageStore.java | 7 +- .../wal/FileWriteAheadLogManager.java | 4 +- .../FsyncModeFileWriteAheadLogManager.java | 4 +- .../cache/persistence/wal/crc/FastCrc.java | 101 ++++++++++++++++++ .../persistence/wal/crc/PureJavaCrc32.java | 2 + .../cache/persistence/wal/io/FileInput.java | 10 +- .../persistence/wal/io/SimpleFileInput.java | 3 +- .../wal/serializer/RecordV1Serializer.java | 5 +- .../db/wal/crc/IgniteDataIntegrityTests.java | 6 +- .../crc/IgnitePureJavaCrcCompatibility.java | 55 ++++++++++ .../testsuites/IgnitePdsTestSuite2.java | 2 + 14 files changed, 284 insertions(+), 21 deletions(-) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java new file mode 100644 index 0000000000000..5c922fead0c80 --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.algo; + +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.nio.ByteBuffer; +import java.util.Random; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.openjdk.jmh.annotations.Mode.AverageTime; +import static org.openjdk.jmh.annotations.Scope.Thread; + +/** + * + */ +@State(Thread) +@OutputTimeUnit(NANOSECONDS) +@BenchmarkMode(AverageTime) +@Fork(value = 1, jvmArgsAppend = {"-XX:+UnlockDiagnosticVMOptions"}) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +public class BenchmarkCRC { + /** */ + static final int SIZE = 1024; + + /** */ + static final int BUF_LEN = 4096; + + /** */ + @State(Thread) + public static class Context { + /** */ + final int[] results = new int[SIZE]; + + /** */ + final ByteBuffer bb = ByteBuffer.allocate(BUF_LEN); + + /** */ + @Setup + public void setup() { + new Random().ints(BUF_LEN, Byte.MIN_VALUE, Byte.MAX_VALUE).forEach(k -> bb.put((byte) k)); + } + } + + /** */ + @Benchmark + public int[] pureJavaCrc32(Context context) { + for (int i = 0; i < SIZE; i++) { + context.bb.rewind(); + + context.results[i] = PureJavaCrc32.calcCrc32(context.bb, BUF_LEN); + } + + return context.results; + } + + /** */ + @Benchmark + public int[] crc32(Context context) { + for (int i = 0; i < SIZE; i++) { + context.bb.rewind(); + + context.results[i] = FastCrc.calcCrc(context.bb, BUF_LEN); + } + + return context.results; + } +} + + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 801bcbd541421..af8bb466635fe 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -132,8 +132,8 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.port.GridPortRecord; import org.apache.ignite.internal.util.GridMultiCollectionWrapper; import org.apache.ignite.internal.util.GridUnsafe; @@ -4134,7 +4134,7 @@ else if (grpId == TxLog.TX_LOG_CACHE_ID) } if (!skipCrc) { - PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize())); + PageIO.setCrc(writeAddr, FastCrc.calcCrc(tmpWriteBuf, pageSize())); tmpWriteBuf.rewind(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java index 008a72830899b..86d9bbc5b56e6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java @@ -21,10 +21,11 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; + +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.spi.encryption.EncryptionSpi; import org.apache.ignite.internal.managers.encryption.GridEncryptionManager; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.util.typedef.internal.CU; /** @@ -260,7 +261,7 @@ private void decrypt(ByteBuffer encrypted, ByteBuffer destBuf) throws IOExceptio * @param res Destination buffer. */ private void storeCRC(ByteBuffer res) { - int crc = PureJavaCrc32.calcCrc32(res, encryptedDataSize()); + int crc = FastCrc.calcCrc(res, encryptedDataSize()); res.put((byte) (crc >> 24)); res.put((byte) (crc >> 16)); @@ -274,7 +275,7 @@ private void storeCRC(ByteBuffer res) { * @param encrypted Encrypted data buffer. */ private void checkCRC(ByteBuffer encrypted) throws IOException { - int crc = PureJavaCrc32.calcCrc32(encrypted, encryptedDataSize()); + int crc = FastCrc.calcCrc(encrypted, encryptedDataSize()); int storedCrc = 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index 2e0786798078b..fdf4705b5b177 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -35,8 +36,8 @@ import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker; import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.util.typedef.internal.U; import static java.nio.file.StandardOpenOption.CREATE; @@ -358,7 +359,7 @@ private long checkFile(FileIO fileIO) throws IOException { pageBuf.position(0); if (!skipCrc) { - int curCrc32 = PureJavaCrc32.calcCrc32(pageBuf, pageSize); + int curCrc32 = FastCrc.calcCrc(pageBuf, pageSize); if ((savedCrc32 ^ curCrc32) != 0) throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " + @@ -613,7 +614,7 @@ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) { try { pageBuf.position(0); - return PureJavaCrc32.calcCrc32(pageBuf, pageSize); + return FastCrc.calcCrc(pageBuf, pageSize); } finally { pageBuf.position(0); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index fe0efbd0e820a..461d38cace499 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -98,8 +98,8 @@ import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; @@ -2439,7 +2439,7 @@ else if (create) buf.position(0); // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos); + int crcVal = FastCrc.calcCrc(buf, curPos); buf.putInt(crcVal); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index 7bfd10bba2376..917640dc026b7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -91,8 +91,8 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO; @@ -2339,7 +2339,7 @@ public static long writeSerializerVersion(FileIO io, long idx, int version, WALM buf.position(0); // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos); + int crcVal = FastCrc.calcCrc(buf, curPos); buf.putInt(crcVal); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java new file mode 100644 index 0000000000000..0dcbafdb9c978 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.crc; + +import java.nio.ByteBuffer; +import java.util.zip.CRC32; + +/** + * This CRC calculation implementation workf much faster then {@link PureJavaCrc32} + */ +public final class FastCrc { + /** CRC algo. */ + private static final ThreadLocal CRC = ThreadLocal.withInitial(CRC32::new); + + /** */ + private final CRC32 crc = new CRC32(); + + /** + * Current value. + */ + private int val; + + /** */ + public FastCrc() { + reset(); + } + + /** + * Preparation for further calculations. + */ + public void reset() { + val = 0xffffffff; + + crc.reset(); + } + + /** + * @return crc value. + */ + public int getValue() { + return val; + } + + /** + * @param buf Input buffer. + * @param len Data length. + */ + public void update(final ByteBuffer buf, final int len) { + val = calcCrc(crc, buf, len); + } + + /** + * @param buf Input buffer. + * @param len Data length. + * + * @return Crc checksum. + */ + public static int calcCrc(ByteBuffer buf, int len) { + CRC32 crcAlgo = CRC.get(); + + int res = calcCrc(crcAlgo, buf, len); + + crcAlgo.reset(); + + return res; + } + + /** + * @param crcAlgo CRC algorithm. + * @param buf Input buffer. + * @param len Buffer length. + * + * @return Crc checksum. + */ + private static int calcCrc(CRC32 crcAlgo, ByteBuffer buf, int len) { + int initLimit = buf.limit(); + + buf.limit(buf.position() + len); + + crcAlgo.update(buf); + + buf.limit(initLimit); + + return (int)crcAlgo.getValue() ^ 0xFFFFFFFF; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java index 6bd4a35b2fdf8..b011f783be005 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java @@ -29,7 +29,9 @@ * succession. * * The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6 + * @deprecated Use {@link FastCrc} instead. */ +@Deprecated public class PureJavaCrc32 { /** * the current CRC value diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java index d19d17b3c7d7f..c9615f597e0f7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java @@ -19,10 +19,12 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.zip.CRC32; + import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.jetbrains.annotations.NotNull; /** @@ -56,7 +58,7 @@ public interface FileInput extends ByteBufferBackedDataInput { */ public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable { /** */ - private final PureJavaCrc32 crc32 = new PureJavaCrc32(); + private final FastCrc crc = new FastCrc(); /** Last calc position. */ private int lastCalcPosition; @@ -93,7 +95,7 @@ public Crc32CheckingFileInput(FileInput delegate, boolean skipCheck) { @Override public void close() throws Exception { updateCrc(); - int val = crc32.getValue(); + int val = crc.getValue(); int writtenCrc = this.readInt(); @@ -118,7 +120,7 @@ private void updateCrc() { buffer().position(lastCalcPosition); - crc32.update(delegate.buffer(), oldPos - lastCalcPosition); + crc.update(delegate.buffer(), oldPos - lastCalcPosition); lastCalcPosition = oldPos; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java index 5918b0b34e93e..1a1562e0909e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java @@ -20,6 +20,7 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; + import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; import org.jetbrains.annotations.NotNull; @@ -264,7 +265,7 @@ private void clearBuffer() { /** * @param skipCheck If CRC check should be skipped. - * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one + * @return autoclosable fileInput, after its closing crc will be calculated and compared with saved one */ public Crc32CheckingFileInput startRead(boolean skipCheck) { return new Crc32CheckingFileInput(this, skipCheck); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index c65f37cc8d648..31efa1f0f1e23 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; + import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.internal.pagemem.wal.WALPointer; @@ -32,6 +33,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; @@ -39,7 +41,6 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; @@ -421,7 +422,7 @@ static void writeWithCrc(WALRecord rec, ByteBuffer buf, RecordIO writer) throws buf.position(startPos); // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos); + int crcVal = FastCrc.calcCrc(buf, curPos - startPos); buf.putInt(crcVal); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java index 59dd3b7e7dc9b..3ad4c9007e752 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java @@ -24,13 +24,14 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.concurrent.ThreadLocalRandom; + import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; /** * @@ -59,6 +60,7 @@ public class IgniteDataIntegrityTests extends TestCase { ); ByteBuffer buf = ByteBuffer.allocate(1024); + ThreadLocalRandom curr = ThreadLocalRandom.current(); for (int i = 0; i < 1024; i+=16) { @@ -66,7 +68,7 @@ public class IgniteDataIntegrityTests extends TestCase { buf.putInt(curr.nextInt()); buf.putInt(curr.nextInt()); buf.position(i); - buf.putInt(PureJavaCrc32.calcCrc32(buf, 12)); + buf.putInt(FastCrc.calcCrc(buf, 12)); } buf.rewind(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java new file mode 100644 index 0000000000000..faafad214270a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc; + +import junit.framework.TestCase; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; + +import java.nio.ByteBuffer; +import java.util.concurrent.ThreadLocalRandom; + +/** + * PureJavaCrc32 previous crc algo realization vs java.util.zip.crc32 test. + */ +public class IgnitePureJavaCrcCompatibility extends TestCase { + /** + * Test crc algo equality results. + * @throws Exception + */ + public void testAlgoEqual() throws Exception { + ByteBuffer buf = ByteBuffer.allocate(1024); + + ThreadLocalRandom curr = ThreadLocalRandom.current(); + + for (int i = 0; i < 1024; i+=16) { + buf.putInt(curr.nextInt()); + buf.putInt(curr.nextInt()); + buf.putInt(curr.nextInt()); + buf.position(i); + + buf.position(i); + int crc0 = FastCrc.calcCrc(buf, 12); + + buf.position(i); + int crc1 = PureJavaCrc32.calcCrc32(buf, 12); + + assertEquals(crc0, crc1); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index bf83bf55dc1e7..ede5cab44f00d 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteFsyncReplayWalIteratorInvalidCrcTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteStandaloneWalIteratorInvalidCrcTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgnitePureJavaCrcCompatibility; import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIteratorTest; @@ -80,6 +81,7 @@ public static TestSuite suite() { suite.addTestSuite(IgniteStandaloneWalIteratorInvalidCrcTest.class); suite.addTestSuite(IgniteReplayWalIteratorInvalidCrcTest.class); suite.addTestSuite(IgniteFsyncReplayWalIteratorInvalidCrcTest.class); + suite.addTestSuite(IgnitePureJavaCrcCompatibility.class); addRealPageStoreTests(suite); From 4482f5c28ae11eb16f85df93d847330b1f696f53 Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Thu, 18 Oct 2018 19:34:55 +0300 Subject: [PATCH 024/403] IGNITE-9155 ExchangeFuture will not complete with Exception on state change failure - Fixes #4467. Signed-off-by: Alexey Goncharuk --- .../GridDhtPartitionsExchangeFuture.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 0fe1a253b7041..4b4b6df7a8862 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3440,8 +3440,6 @@ else if (forceAffReassignment) } if (stateChangeExchange()) { - IgniteCheckedException err = null; - StateChangeRequest req = exchActions.stateChangeRequest(); assert req != null : exchActions; @@ -3451,8 +3449,6 @@ else if (forceAffReassignment) if (!F.isEmpty(exchangeGlobalExceptions)) { stateChangeErr = true; - err = new IgniteCheckedException("Cluster state change failed."); - cctx.kernalContext().state().onStateChangeError(exchangeGlobalExceptions, req); } else { @@ -3484,7 +3480,7 @@ else if (forceAffReassignment) cctx.discovery().sendCustomEvent(stateFinishMsg); if (!centralizedAff) - onDone(exchCtx.events().topologyVersion(), err); + onDone(exchCtx.events().topologyVersion(), null); } } catch (IgniteCheckedException e) { @@ -3969,15 +3965,10 @@ else if (forceAffReassignment) updatePartitionFullMap(resTopVer, msg); - IgniteCheckedException err = null; - - if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap())) { - err = new IgniteCheckedException("Cluster state change failed"); - + if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap())) cctx.kernalContext().state().onStateChangeError(msg.getErrorsMap(), exchActions.stateChangeRequest()); - } - onDone(resTopVer, err); + onDone(resTopVer, null); } catch (IgniteCheckedException e) { onDone(e); From eb06163be1e0af741dc7692e95989d053be6bf0a Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Fri, 19 Oct 2018 10:39:17 +0300 Subject: [PATCH 025/403] IGNITE-9875 Added missing licence --- ...ridDhtPartitionsStateValidatorBenchmark.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java index 151606dab21ba..f3bbcb96d6cb9 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.benchmarks.jmh.misc; import com.google.common.collect.Lists; From 33c477a3b3746a288f9d62cad1a6806c4ce4ca65 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Fri, 19 Oct 2018 11:44:38 +0300 Subject: [PATCH 026/403] IGNITE-9785 Fixed merge artifacts --- .../datastreamer/DataStreamerImpl.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index e86f65313d21a..4253fb28a3317 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -2206,24 +2206,9 @@ protected static class IsolatedUpdater implements StreamReceiver cctx = internalCache.context(); -/* todo: uncomment this and remove topFut choosing logic below after IGNITE-9550 race is fixed GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture(); AffinityTopologyVersion topVer = topFut.topologyVersion(); -*/ - AffinityTopologyVersion topVer = cctx.isLocal() ? - cctx.affinity().affinityTopologyVersion() : - cctx.shared().exchange().readyAffinityVersion(); - - GridDhtTopologyFuture topFut = (GridDhtTopologyFuture)cctx.shared().exchange().affinityReadyFuture(topVer); - - if (topFut == null) { - // Exchange for newer topology version is already in progress, let's try to use last finished future. - GridDhtTopologyFuture lastFinishedFut = cctx.shared().exchange().lastFinishedFuture(); - - if (F.eq(lastFinishedFut.topologyVersion(), topVer)) - topFut = lastFinishedFut; - } GridCacheVersion ver = cctx.versions().isolatedStreamerVersion(); @@ -2285,8 +2270,6 @@ else if (ttl == CU.TTL_NOT_CHANGED) } if (topFut != null) { - topFut.get(); - Throwable err = topFut.validateCache(cctx, false, false, entry.key(), null); if (err != null) From e52752a2319cfadf8ef794dd907bae5915ce6137 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Fri, 19 Oct 2018 17:51:41 +0300 Subject: [PATCH 027/403] IGNITE-5795 Register binary metadata during cache start - Fixes #4852. Signed-off-by: Alexey Goncharuk --- .../apache/ignite/internal/IgniteKernal.java | 2 +- .../binary/BinaryCachingMetadataHandler.java | 25 +- .../ignite/internal/binary/BinaryContext.java | 59 +++- .../binary/BinaryMetadataHandler.java | 10 + .../binary/BinaryNoopMetadataHandler.java | 6 + .../builder/BinaryObjectBuilderImpl.java | 2 +- .../internal/client/thin/TcpIgniteClient.java | 6 + .../processors/cache/GridCacheProcessor.java | 9 +- .../CacheObjectBinaryProcessorImpl.java | 5 + .../processors/query/GridQueryProcessor.java | 77 ++++- .../binary/TestCachingMetadataHandler.java | 6 + .../CacheRegisterMetadataLocallyTest.java | 287 ++++++++++++++++++ .../cache/index/AbstractSchemaSelfTest.java | 6 +- .../index/H2DynamicIndexAbstractSelfTest.java | 48 +-- .../IgniteCacheWithIndexingTestSuite.java | 2 + 15 files changed, 494 insertions(+), 56 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index c6ec9be1e1848..40347d71473e6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1005,6 +1005,7 @@ public void start( // Start processors before discovery manager, so they will // be able to start receiving messages once discovery completes. try { + startProcessor(new GridMarshallerMappingProcessor(ctx)); startProcessor(new PdsConsistentIdProcessor(ctx)); startProcessor(new MvccProcessorImpl(ctx)); startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx)); @@ -1028,7 +1029,6 @@ public void start( startProcessor(createHadoopComponent()); startProcessor(new DataStructuresProcessor(ctx)); startProcessor(createComponent(PlatformProcessor.class, ctx)); - startProcessor(new GridMarshallerMappingProcessor(ctx)); // Start plugins. for (PluginProvider provider : ctx.plugins().allProviders()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java index a0559cbdc8ba1..b60dc097aaa1e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java @@ -46,23 +46,28 @@ private BinaryCachingMetadataHandler() { } /** {@inheritDoc} */ - @Override public synchronized void addMeta(int typeId, BinaryType type, boolean failIfUnregistered) throws BinaryObjectException { - synchronized (this) { - BinaryType oldType = metas.put(typeId, type); + @Override public synchronized void addMeta(int typeId, BinaryType type, + boolean failIfUnregistered) throws BinaryObjectException { + BinaryType oldType = metas.put(typeId, type); - if (oldType != null) { - BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata(); - BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata(); + if (oldType != null) { + BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata(); + BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata(); - BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta); + BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta); - BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context()); + BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context()); - metas.put(typeId, mergedType); - } + metas.put(typeId, mergedType); } } + /** {@inheritDoc} */ + @Override public synchronized void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) + throws BinaryObjectException { + addMeta(typeId, meta, failIfUnregistered); + } + /** {@inheritDoc} */ @Override public synchronized BinaryType metadata(int typeId) throws BinaryObjectException { return metas.get(typeId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java index 7885d9575f32b..7ab74e09b82d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java @@ -617,6 +617,18 @@ else if (cpElement.isFile()) { */ public BinaryClassDescriptor descriptorForClass(Class cls, boolean deserialize, boolean failIfUnregistered) throws BinaryObjectException { + return descriptorForClass(cls, deserialize, failIfUnregistered, false); + } + + /** + * @param cls Class. + * @param failIfUnregistered Throw exception if class isn't registered. + * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all. + * @return Class descriptor. + * @throws BinaryObjectException In case of error. + */ + public BinaryClassDescriptor descriptorForClass(Class cls, boolean deserialize, boolean failIfUnregistered, + boolean onlyLocReg) throws BinaryObjectException { assert cls != null; BinaryClassDescriptor desc = descByCls.get(cls); @@ -625,7 +637,7 @@ public BinaryClassDescriptor descriptorForClass(Class cls, boolean deserializ if (failIfUnregistered) throw new UnregisteredClassException(cls); - desc = registerClassDescriptor(cls, deserialize); + desc = registerClassDescriptor(cls, deserialize, onlyLocReg); } else if (!desc.registered()) { if (!desc.userType()) { @@ -662,7 +674,7 @@ else if (!desc.registered()) { if (failIfUnregistered) throw new UnregisteredClassException(cls); - desc = registerUserClassDescriptor(desc); + desc = registerUserClassDescriptor(desc, onlyLocReg); } } @@ -715,7 +727,7 @@ public BinaryClassDescriptor descriptorForTypeId( } if (desc == null) { - desc = registerClassDescriptor(cls, deserialize); + desc = registerClassDescriptor(cls, deserialize, false); assert desc.typeId() == typeId : "Duplicate typeId [typeId=" + typeId + ", cls=" + cls + ", desc=" + desc + "]"; @@ -728,9 +740,10 @@ public BinaryClassDescriptor descriptorForTypeId( * Creates and registers {@link BinaryClassDescriptor} for the given {@code class}. * * @param cls Class. + * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all. * @return Class descriptor. */ - private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean deserialize) { + private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean deserialize, boolean onlyLocReg) { BinaryClassDescriptor desc; String clsName = cls.getName(); @@ -759,7 +772,7 @@ private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean dese desc = old; } else - desc = registerUserClassDescriptor(cls, deserialize); + desc = registerUserClassDescriptor(cls, deserialize, onlyLocReg); return desc; } @@ -768,9 +781,10 @@ private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean dese * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}. * * @param cls Class. + * @param onlyLocReg {@code true} if descriptor need to register only locally. * @return Class descriptor. */ - private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean deserialize) { + private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean deserialize, boolean onlyLocReg) { boolean registered; final String clsName = cls.getName(); @@ -781,7 +795,7 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean final int typeId = mapper.typeId(clsName); - registered = registerUserClassName(typeId, cls.getName()); + registered = registerUserClassName(typeId, cls.getName(), onlyLocReg); BinarySerializer serializer = serializerForClass(cls); @@ -799,9 +813,22 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean registered ); - if (!deserialize) - metaHnd.addMeta(typeId, new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), affFieldName, null, - desc.isEnum(), cls.isEnum() ? enumMap(cls) : null).wrap(this), false); + if (!deserialize) { + BinaryMetadata binaryMetadata = new BinaryMetadata( + typeId, + typeName, + desc.fieldsMeta(), + affFieldName, + null, + desc.isEnum(), + cls.isEnum() ? enumMap(cls) : null + ); + + if (onlyLocReg) + metaHnd.addMetaLocally(typeId, binaryMetadata.wrap(this), false); + else + metaHnd.addMeta(typeId, binaryMetadata.wrap(this), false); + } descByCls.put(cls, desc); @@ -814,12 +841,13 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}. * * @param desc Old descriptor that should be re-registered. + * @param onlyLocReg {@code true} if descriptor need to register only locally. * @return Class descriptor. */ - private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc) { + private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc, boolean onlyLocReg) { boolean registered; - registered = registerUserClassName(desc.typeId(), desc.describedClass().getName()); + registered = registerUserClassName(desc.typeId(), desc.describedClass().getName(), onlyLocReg); if (registered) { BinarySerializer serializer = desc.initialSerializer(); @@ -1191,15 +1219,18 @@ public void registerUserTypesSchema() { * * @param typeId Type ID. * @param clsName Class Name. + * @param onlyLocReg {@code true} if descriptor need to register only locally. * @return {@code True} if the mapping was registered successfully. */ - public boolean registerUserClassName(int typeId, String clsName) { + public boolean registerUserClassName(int typeId, String clsName, boolean onlyLocReg) { IgniteCheckedException e = null; boolean res = false; try { - res = marshCtx.registerClassName(JAVA_ID, typeId, clsName); + res = onlyLocReg + ? marshCtx.registerClassNameLocally(JAVA_ID, typeId, clsName) + : marshCtx.registerClassName(JAVA_ID, typeId, clsName); } catch (DuplicateTypeIdException dupEx) { // Ignore if trying to register mapped type name of the already registered class name and vise versa diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java index 85ab1372f49f2..d1336bf6ae4a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java @@ -35,6 +35,16 @@ public interface BinaryMetadataHandler { */ public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException; + /** + * Adds meta data locally on current node without sending any messages. + * + * @param typeId Type ID. + * @param meta Metadata. + * @param failIfUnregistered Fail if unregistered. + * @throws BinaryObjectException In case of error. + */ + public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException; + /** * Gets meta data for provided type ID. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java index 4ee24285c7ee5..a552d611ea990 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java @@ -47,6 +47,12 @@ private BinaryNoopMetadataHandler() { // No-op. } + /** {@inheritDoc} */ + @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) + throws BinaryObjectException { + // No-op. + } + /** {@inheritDoc} */ @Override public BinaryType metadata(int typeId) throws BinaryObjectException { return null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index abd63cd7b53ca..5414a25f3fa63 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -364,7 +364,7 @@ else if (readCache == null) { if (affFieldName0 == null) affFieldName0 = ctx.affinityKeyFieldName(typeId); - ctx.registerUserClassName(typeId, typeName); + ctx.registerUserClassName(typeId, typeName, false); ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldsMeta, affFieldName0, Collections.singleton(curSchema), false, null), writer.failIfUnregistered()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java index 50408168d3118..856f41c4a324b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java @@ -262,6 +262,12 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler { cache.addMeta(typeId, meta, failIfUnregistered); // merge } + /** {@inheritDoc} */ + @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) + throws BinaryObjectException { + throw new UnsupportedOperationException("Can't register metadata locally for thin client."); + } + /** {@inheritDoc} */ @Override public BinaryType metadata(int typeId) throws BinaryObjectException { BinaryType meta = cache.metadata(typeId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index ec88a938893c5..4a6bed4f7a51e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -3823,8 +3823,13 @@ else if (msg0 instanceof WalStateFinishMessage) return msg0.needExchange(); } - if (msg instanceof DynamicCacheChangeBatch) - return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer); + if (msg instanceof DynamicCacheChangeBatch) { + boolean changeRequested = cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer); + + ctx.query().onCacheChangeRequested((DynamicCacheChangeBatch)msg); + + return changeRequested; + } if (msg instanceof DynamicCacheChangeFailureMessage) cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg, topVer); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index 137db9f887dc5..f3078cb18df80 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -207,6 +207,11 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) { CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx), failIfUnregistered); } + @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) + throws BinaryObjectException { + CacheObjectBinaryProcessorImpl.this.addMetaLocally(typeId, meta); + } + @Override public BinaryType metadata(int typeId) throws BinaryObjectException { return CacheObjectBinaryProcessorImpl.this.metadata(typeId); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index b5b104d2c2f51..366c62a240c9c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.query; +import javax.cache.Cache; +import javax.cache.CacheException; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; @@ -34,8 +36,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; -import javax.cache.Cache; -import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; @@ -62,16 +62,20 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; +import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch; +import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture; import org.apache.ignite.internal.processors.cache.query.CacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; +import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.query.property.QueryBinaryProperty; import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheFilter; import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor; @@ -257,6 +261,8 @@ public GridQueryProcessor(GridKernalContext ctx) throws IgniteCheckedException { ctxs.queries().evictDetailMetrics(); } }, QRY_DETAIL_METRICS_EVICTION_FREQ, QRY_DETAIL_METRICS_EVICTION_FREQ); + + registerMetadataForRegisteredCaches(); } /** {@inheritDoc} */ @@ -903,6 +909,73 @@ public void skipFieldLookup(boolean skipFieldLookup) { this.skipFieldLookup = skipFieldLookup; } + /** + * Register metadata locally for already registered caches. + */ + private void registerMetadataForRegisteredCaches() { + for (DynamicCacheDescriptor cacheDescriptor : ctx.cache().cacheDescriptors().values()) { + registerBinaryMetadata(cacheDescriptor.cacheConfiguration(), cacheDescriptor.schema()); + } + } + + /** + * Handle of cache change request. + * + * @param batch Dynamic cache change batch request. + */ + public void onCacheChangeRequested(DynamicCacheChangeBatch batch) { + for (DynamicCacheChangeRequest req : batch.requests()) { + if (!req.start()) + continue; + + registerBinaryMetadata(req.startCacheConfiguration(), req.schema()); + } + } + + /** + * Register binary metadata locally. + * + * @param ccfg Cache configuration. + * @param schema Schema for which register metadata is required. + */ + private void registerBinaryMetadata(CacheConfiguration ccfg, QuerySchema schema) { + if (schema != null) { + Collection qryEntities = schema.entities(); + + if (!F.isEmpty(qryEntities)) { + boolean binaryEnabled = ctx.cacheObjects().isBinaryEnabled(ccfg); + + if (binaryEnabled) { + for (QueryEntity qryEntity : qryEntities) { + Class keyCls = U.box(U.classForName(qryEntity.findKeyType(), null, true)); + Class valCls = U.box(U.classForName(qryEntity.findValueType(), null, true)); + + if (keyCls != null) + registerDescriptorLocallyIfNeeded(keyCls); + + if (valCls != null) + registerDescriptorLocallyIfNeeded(valCls); + } + } + } + } + } + + /** + * Register class metadata locally if it didn't do it earlier. + * + * @param cls Class for which the metadata should be registered. + */ + private void registerDescriptorLocallyIfNeeded(Class cls) { + IgniteCacheObjectProcessor cacheObjProc = ctx.cacheObjects(); + + if (cacheObjProc instanceof CacheObjectBinaryProcessorImpl) { + ((CacheObjectBinaryProcessorImpl)cacheObjProc) + .binaryContext() + .descriptorForClass(cls, false, false, true); + } + } + /** * Handle custom discovery message. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java index c515f8191766a..47138ddeb0806 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java @@ -38,6 +38,12 @@ public class TestCachingMetadataHandler implements BinaryMetadataHandler { TestCachingMetadataHandler.class.getSimpleName() + '.'); } + /** {@inheritDoc} */ + @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) + throws BinaryObjectException { + addMeta(typeId, meta, failIfUnregistered); + } + /** {@inheritDoc} */ @Override public BinaryType metadata(int typeId) throws BinaryObjectException { return metas.get(typeId); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java new file mode 100644 index 0000000000000..d4066c2f546f6 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.Collections; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.binary.BinaryType; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper; +import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage; +import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage; +import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests, that binary metadata is registered correctly during the start without extra request to grid. + */ +public class CacheRegisterMetadataLocallyTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static final String STATIC_CACHE_NAME = "staticCache"; + + /** */ + private static final String DYNAMIC_CACHE_NAME = "dynamicCache"; + + /** Holder of sent custom messages. */ + private final ConcurrentLinkedQueue customMessages = new ConcurrentLinkedQueue<>(); + + /** Holder of sent communication messages. */ + private final ConcurrentLinkedQueue communicationMessages = new ConcurrentLinkedQueue<>(); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDiscoverySpi(new TcpDiscoverySpi() { + @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException { + if (msg instanceof CustomMessageWrapper) + customMessages.add(((CustomMessageWrapper)msg).delegate()); + else + customMessages.add(msg); + + super.sendCustomEvent(msg); + } + }); + + cfg.setCommunicationSpi(new TcpCommunicationSpi() { + @Override public void sendMessage(ClusterNode node, Message msg, + IgniteInClosure ackC) throws IgniteSpiException { + if (msg instanceof GridIoMessage) + communicationMessages.add(((GridIoMessage)msg).message()); + + super.sendMessage(node, msg, ackC); + } + + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + if (msg instanceof GridIoMessage) + communicationMessages.add(((GridIoMessage)msg).message()); + + super.sendMessage(node, msg); + } + }); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + if (igniteInstanceName.equals("client")) + cfg.setClientMode(true); + + cfg.setCacheConfiguration(cacheConfiguration(STATIC_CACHE_NAME, StaticKey.class, StaticValue.class)); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + customMessages.clear(); + communicationMessages.clear(); + } + + /** + * @throws Exception If failed. + */ + public void testAffinityKeyRegisteredStaticCache() throws Exception { + Ignite ignite = startGrid(); + + assertEquals("affKey", getAffinityKey(ignite, StaticKey.class)); + assertEquals("affKey", getAffinityKey(ignite, StaticValue.class)); + } + + /** + * @throws Exception If failed. + */ + public void testAffinityKeyRegisteredDynamicCache() throws Exception { + Ignite ignite = startGrid(); + + ignite.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class)); + + assertEquals("affKey", getAffinityKey(ignite, DynamicKey.class)); + assertEquals("affKey", getAffinityKey(ignite, DynamicValue.class)); + } + + /** + * @throws Exception If failed. + */ + public void testClientFindsValueByAffinityKeyStaticCacheWithoutExtraRequest() throws Exception { + Ignite srv = startGrid(); + IgniteCache cache = srv.cache(STATIC_CACHE_NAME); + + testClientFindsValueByAffinityKey(cache, new StaticKey(1), new StaticValue(2)); + + assertCustomMessages(2); //MetadataUpdateProposedMessage for update schema. + assertCommunicationMessages(); + } + + /** + * @throws Exception If failed. + */ + public void testClientFindsValueByAffinityKeyDynamicCacheWithoutExtraRequest() throws Exception { + Ignite srv = startGrid(); + IgniteCache cache = + srv.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class)); + + testClientFindsValueByAffinityKey(cache, new DynamicKey(3), new DynamicValue(4)); + + //Expected only DynamicCacheChangeBatch for start cache and MetadataUpdateProposedMessage for update schema. + assertCustomMessages(3); + assertCommunicationMessages(); + } + + /** + * @param ignite Ignite instance. + * @param keyCls Key class. + * @return Name of affinity key field of the given class. + */ + private String getAffinityKey(Ignite ignite, Class keyCls) { + BinaryType binType = ignite.binary().type(keyCls); + + return binType.affinityKeyFieldName(); + } + + /** + * @param cache Cache instance. + * @param key Test key. + * @param val Test value. + * @throws Exception If failed. + */ + private void testClientFindsValueByAffinityKey(IgniteCache cache, K key, V val) throws Exception { + cache.put(key, val); + + assertTrue(cache.containsKey(key)); + + Ignite client = startGrid("client"); + + IgniteCache clientCache = client.cache(cache.getName()); + + assertTrue(clientCache.containsKey(key)); + } + + /** + * @param name Cache name. + * @param keyCls Key {@link Class}. + * @param valCls Value {@link Class}. + * @param Key type. + * @param Value type. + * @return Cache configuration + */ + private static CacheConfiguration cacheConfiguration(String name, Class keyCls, Class valCls) { + CacheConfiguration cfg = new CacheConfiguration<>(name); + cfg.setQueryEntities(Collections.singleton(new QueryEntity(keyCls, valCls))); + return cfg; + } + + /** + * Expecting that "proposed binary metadata"( {@link org.apache.ignite.internal.processors.marshaller.MappingProposedMessage}, + * {@link org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage}) will be skipped because + * it should be register locally during the start. + * + * @param expMsgCnt Count of expected messages. + */ + private void assertCustomMessages(int expMsgCnt) { + assertEquals(customMessages.toString(), expMsgCnt, customMessages.size()); + + customMessages.forEach(cm -> assertTrue(cm.toString(), cm instanceof DynamicCacheChangeBatch || cm instanceof MetadataUpdateProposedMessage)); + } + + /** + * Expecting that extra request to binary metadata( {@link MetadataRequestMessage}, {@link MetadataResponseMessage}) + * will be skipped because it should be register locally during the start. + */ + private void assertCommunicationMessages() { + communicationMessages.forEach(cm -> + assertFalse(cm.toString(), cm instanceof MetadataRequestMessage || cm instanceof MetadataResponseMessage) + ); + } + + /** */ + private static class StaticKey { + /** */ + @AffinityKeyMapped + private int affKey; + + /** + * @param affKey Affinity key. + */ + StaticKey(int affKey) { + this.affKey = affKey; + } + } + + /** */ + private static class StaticValue { + /** */ + @AffinityKeyMapped + private int affKey; + + /** + * @param affKey Affinity key. + */ + StaticValue(int affKey) { + } + } + + /** */ + private static class DynamicKey { + /** */ + @AffinityKeyMapped + private int affKey; + + /** + * @param affKey Affinity key. + */ + DynamicKey(int affKey) { + this.affKey = affKey; + } + } + + /** */ + private static class DynamicValue { + /** */ + @AffinityKeyMapped + private int affKey; + + /** + * @param affKey Affinity key. + */ + DynamicValue(int affKey) { + this.affKey = affKey; + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java index 7f1e2e74e8ec1..5f0e18e900617 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java @@ -629,21 +629,21 @@ public long id() { public static class ValueClass { /** Field 1. */ @QuerySqlField - private String field1; + private Long field1; /** * Constructor. * * @param field1 Field 1. */ - public ValueClass(String field1) { + public ValueClass(Long field1) { this.field1 = field1; } /** * @return Field 1 */ - public String field1() { + public Long field1() { return field1; } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java index 10ef56fcd3b5e..07cb96b3c17de 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java @@ -60,9 +60,9 @@ public abstract class H2DynamicIndexAbstractSelfTest extends AbstractSchemaSelfT IgniteCache cache = client().cache(CACHE_NAME); - cache.put(new KeyClass(1), new ValueClass("val1")); - cache.put(new KeyClass(2), new ValueClass("val2")); - cache.put(new KeyClass(3), new ValueClass("val3")); + cache.put(new KeyClass(1), new ValueClass(1L)); + cache.put(new KeyClass(2), new ValueClass(2L)); + cache.put(new KeyClass(3), new ValueClass(3L)); } /** {@inheritDoc} */ @@ -89,14 +89,14 @@ public void testCreateIndex() throws Exception { continue; List> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " + - "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll(); + "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll(); assertEquals(F.asList( Collections.singletonList("SELECT\n" + " \"id\"\n" + "FROM \"cache\".\"ValueClass\"\n" + - " /* \"cache\".\"idx_1\": \"field1\" = 'A' */\n" + - "WHERE \"field1\" = 'A'") + " /* \"cache\".\"idx_1\": \"field1\" = 1 */\n" + + "WHERE \"field1\" = 1") ), locRes); } @@ -106,7 +106,7 @@ public void testCreateIndex() throws Exception { assertSize(2); - cache.put(new KeyClass(4), new ValueClass("someVal")); + cache.put(new KeyClass(4), new ValueClass(1L)); assertSize(3); } @@ -162,14 +162,14 @@ public void testDropIndex() { continue; List> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " + - "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll(); + "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll(); assertEquals(F.asList( Collections.singletonList("SELECT\n" + " \"id\"\n" + "FROM \"cache\".\"ValueClass\"\n" + " /* \"cache\".\"ValueClass\".__SCAN_ */\n" + - "WHERE \"field1\" = 'A'") + "WHERE \"field1\" = 1") ), locRes); } @@ -204,38 +204,39 @@ public void testDropMissingIndexIfExists() { public void testIndexState() { IgniteCache cache = cache(); - assertColumnValues("val1", "val2", "val3"); + assertColumnValues(1L, 2L, 3L); cache.query(new SqlFieldsQuery("CREATE INDEX \"" + IDX_NAME_1_ESCAPED + "\" ON \"" + TBL_NAME_ESCAPED + "\"(\"" + FIELD_NAME_1_ESCAPED + "\" ASC)")); - assertColumnValues("val1", "val2", "val3"); + assertColumnValues(1L, 2L, 3L); cache.remove(new KeyClass(2)); - assertColumnValues("val1", "val3"); + assertColumnValues(1L, 3L); - cache.put(new KeyClass(0), new ValueClass("someVal")); + cache.put(new KeyClass(0), new ValueClass(0L)); - assertColumnValues("someVal", "val1", "val3"); + assertColumnValues(0L, 1L, 3L); cache.query(new SqlFieldsQuery("DROP INDEX \"" + IDX_NAME_1_ESCAPED + "\"")); - assertColumnValues("someVal", "val1", "val3"); + assertColumnValues(0L, 1L, 3L); } /** * Check that values of {@code field1} match what we expect. * @param vals Expected values. */ - private void assertColumnValues(String... vals) { + private void assertColumnValues(Long... vals) { List> expRes = new ArrayList<>(vals.length); - for (String v : vals) + for (Long v : vals) expRes.add(Collections.singletonList(v)); - assertEquals(expRes, cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" + - TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll()); + List> all = cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" + + TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll(); + assertEquals(expRes, all); } /** @@ -245,8 +246,9 @@ private void assertColumnValues(String... vals) { private void assertSize(long expSize) { assertEquals(expSize, cache().size()); - assertEquals(expSize, cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\"")) - .getAll().get(0).get(0)); + Object actual = cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\"")) + .getAll().get(0).get(0); + assertEquals(expSize, actual); } /** @@ -313,8 +315,8 @@ private CacheConfiguration cacheConfiguration() { entity.setValueType(ValueClass.class.getName()); entity.addQueryField("id", Long.class.getName(), null); - entity.addQueryField(FIELD_NAME_1_ESCAPED, String.class.getName(), null); - entity.addQueryField(FIELD_NAME_2_ESCAPED, String.class.getName(), null); + entity.addQueryField(FIELD_NAME_1_ESCAPED, Long.class.getName(), null); + entity.addQueryField(FIELD_NAME_2_ESCAPED, Long.class.getName(), null); entity.setKeyFields(Collections.singleton("id")); diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index 8517ebbf2a503..fae196f1df4c8 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -19,6 +19,7 @@ import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.BinaryTypeMismatchLoggingTest; +import org.apache.ignite.internal.processors.cache.CacheRegisterMetadataLocallyTest; import org.apache.ignite.internal.processors.cache.CacheBinaryKeyConcurrentQueryTest; import org.apache.ignite.internal.processors.cache.CacheConfigurationP2PTest; import org.apache.ignite.internal.processors.cache.CacheIndexStreamerTest; @@ -80,6 +81,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(CacheOperationsWithExpirationTest.class); suite.addTestSuite(CacheBinaryKeyConcurrentQueryTest.class); suite.addTestSuite(CacheQueryFilterExpiredTest.class); + suite.addTestSuite(CacheRegisterMetadataLocallyTest.class); suite.addTestSuite(ClientReconnectAfterClusterRestartTest.class); From 084388dbb4bed5427ce5071d2c862664b1835b18 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Mon, 22 Oct 2018 16:01:28 +0300 Subject: [PATCH 028/403] IGNITE-9945 Entry start version should not omit datacenter ID - Fixes #5034. Signed-off-by: Alexey Goncharuk (cherry picked from commit 4353618a7501e2dada6b0027cadc2b9c6b069fde) --- .../cache/CacheConflictResolutionManager.java | 2 +- .../processors/cache/GridCacheMapEntry.java | 9 ++---- .../version/GridCacheVersionManager.java | 31 +++++++++++++++++-- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java index 6d65d828fc298..9790f754a8d5a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java @@ -17,7 +17,7 @@ package org.apache.ignite.internal.processors.cache; -import org.apache.ignite.internal.processors.cache.version.*; +import org.apache.ignite.internal.processors.cache.version.CacheVersionConflictResolver; /** * Conflict resolver manager. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 9bb8aec1b3ba1..87de4b9d358c1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -73,7 +73,6 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; -import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx; import org.apache.ignite.internal.processors.dr.GridDrType; import org.apache.ignite.internal.processors.query.IgniteSQLException; @@ -117,10 +116,8 @@ import static org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome.REMOVE_NO_VAL; import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.compareIgnoreOpCounter; import static org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.RowData.NO_KEY; -import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.CONCURRENT_UPDATE; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.DUPLICATE_KEY; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.TRANSACTION_SERIALIZATION_ERROR; -import static org.apache.ignite.internal.processors.dr.GridDrType.DR_BACKUP; import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE; /** @@ -222,7 +219,7 @@ protected GridCacheMapEntry( this.cctx = cctx; this.listenerLock = cctx.continuousQueries().getListenerReadLock(); - ver = GridCacheVersionManager.START_VER; + ver = cctx.shared().versions().startVersion(); } /** @@ -332,7 +329,7 @@ protected void value(@Nullable CacheObject val) { * @return {@code True} if start version. */ public boolean isStartVersion() { - return ver == GridCacheVersionManager.START_VER; + return cctx.shared().versions().isStartVersion(ver); } /** {@inheritDoc} */ @@ -3292,7 +3289,7 @@ protected final boolean hasValueUnlocked() { GridCacheVersion currentVer = row != null ? row.version() : GridCacheMapEntry.this.ver; - boolean isStartVer = currentVer == GridCacheVersionManager.START_VER; + boolean isStartVer = cctx.shared().versions().isStartVersion(currentVer); if (cctx.group().persistenceEnabled()) { if (!isStartVer) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java index df8af4811b7b3..2a6526aa2a377 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java @@ -41,9 +41,6 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { /** */ public static final GridCacheVersion EVICT_VER = new GridCacheVersion(Integer.MAX_VALUE, 0, 0, 0); - /** */ - public static final GridCacheVersion START_VER = new GridCacheVersion(0, 0, 0, 0); - /** Timestamp used as base time for cache topology version (January 1, 2014). */ public static final long TOP_VER_BASE_TIME = 1388520000000L; @@ -56,6 +53,9 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { /** Current order for store operations. */ private final AtomicLong loadOrder = new AtomicLong(0); + /** Entry start version. */ + private GridCacheVersion startVer; + /** Last version. */ private volatile GridCacheVersion last; @@ -87,6 +87,8 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { @Override public void start0() throws IgniteCheckedException { last = new GridCacheVersion(0, order.get(), 0, dataCenterId); + startVer = new GridCacheVersion(0, 0, 0, dataCenterId); + cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_METRICS_UPDATED); } @@ -104,6 +106,8 @@ public void dataCenterId(byte dataCenterId) { this.dataCenterId = dataCenterId; last = new GridCacheVersion(0, order.get(), 0, dataCenterId); + + startVer = new GridCacheVersion(0, 0, 0, dataCenterId); } /** @@ -301,4 +305,25 @@ private GridCacheVersion next(long topVer, boolean addTime, boolean forLoad, byt public GridCacheVersion last() { return last; } + + /** + * Gets start version. + * + * @return Start version. + */ + public GridCacheVersion startVersion() { + assert startVer != null; + + return startVer; + } + + /** + * Check if given version is start version. + * + * @param ver Version. + * @return {@code True} if given version is start version. + */ + public boolean isStartVersion(GridCacheVersion ver) { + return startVer.equals(ver); + } } From 2645fd995a67bbdef4a8ee6ee38760551d2a965f Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Mon, 22 Oct 2018 16:10:47 +0300 Subject: [PATCH 029/403] IGNITE-9945 Entry start version should not omit datacenter ID. (cherry picked from commit e5a467272f88e5561a4cc30b510a5608a3b5e739) --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 2 +- .../internal/processors/cache/dr/GridCacheDrManager.java | 3 ++- .../internal/processors/cache/dr/GridOsCacheDrManager.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 4b4b6df7a8862..d4948576737f6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2105,7 +2105,7 @@ public void finishMerged() { if (drCacheCtx.isDrEnabled()) { try { - drCacheCtx.dr().onExchange(res, exchId.isLeft()); + drCacheCtx.dr().onExchange(res, exchId.isLeft(), activateCluster()); } catch (IgniteCheckedException e) { U.error(log, "Failed to notify DR: " + e, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java index f2a4b30c4af82..33a52a11a03d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java @@ -60,9 +60,10 @@ public void replicate(KeyCacheObject key, * * @param topVer Topology version. * @param left {@code True} if exchange has been caused by node leave. + * @param activate {@code True} if exchange has been caused by cluster activation. * @throws IgniteCheckedException If failed. */ - public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException; + public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException; /** * @return {@code True} is DR is enabled. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java index f3c1b23f7c7d6..425e79c536344 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java @@ -78,7 +78,7 @@ public class GridOsCacheDrManager implements GridCacheDrManager { } /** {@inheritDoc} */ - @Override public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException { + @Override public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException { // No-op. } From be2efd7a8b976c19e36694d72f495b936b7d47d9 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Mon, 22 Oct 2018 16:27:49 +0300 Subject: [PATCH 030/403] IGNITE-8006 Parallelize cache groups start - Fixes #4752. --- .../apache/ignite/IgniteSystemProperties.java | 7 + .../cache/CacheAffinitySharedManager.java | 115 ++-- .../processors/cache/GridCacheIoManager.java | 22 +- .../GridCachePartitionExchangeManager.java | 4 +- .../processors/cache/GridCacheProcessor.java | 523 +++++++++++++----- .../processors/cache/StartCacheInfo.java | 113 ++++ .../GridDhtPartitionsExchangeFuture.java | 78 ++- .../ignite/internal/util/IgniteUtils.java | 104 ++-- .../util/InitializationProtector.java | 79 +++ .../util/lang/IgniteThrowableConsumer.java | 37 ++ .../util/lang/IgniteThrowableRunner.java | 30 + .../distributed/CacheStartInParallelTest.java | 219 ++++++++ .../IgniteCrossCacheTxStoreSelfTest.java | 44 +- .../internal/util/IgniteUtilsSelfTest.java | 74 +++ .../testsuites/IgniteCacheTestSuite7.java | 3 + 15 files changed, 1165 insertions(+), 287 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 5e2297c9d2143..99ed040e82d9c 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1029,6 +1029,13 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT = "IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT"; + /** + * Enables start caches in parallel. + * + * Default is {@code true}. + */ + public static final String IGNITE_ALLOW_START_CACHES_IN_PARALLEL = "IGNITE_ALLOW_START_CACHES_IN_PARALLEL"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index cedbde1c75f9e..6e10c0009887f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -17,12 +17,14 @@ package org.apache.ignite.internal.processors.cache; +import javax.cache.CacheException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -30,7 +32,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; @@ -427,25 +428,43 @@ void onCacheGroupCreated(CacheGroupContext grp) { Map fetchFuts = U.newHashMap(startDescs.size()); - Set startedCaches = U.newHashSet(startDescs.size()); - Map startedInfos = U.newHashMap(startDescs.size()); - for (DynamicCacheDescriptor desc : startDescs) { - try { - startedCaches.add(desc.cacheName()); + List startCacheInfos = startDescs.stream() + .map(desc -> { + DynamicCacheChangeRequest changeReq = startReqs.get(desc.cacheName()); - DynamicCacheChangeRequest startReq = startReqs.get(desc.cacheName()); - - cctx.cache().prepareCacheStart( - desc.cacheConfiguration(), + return new StartCacheInfo( desc, - startReq.nearCacheConfiguration(), + changeReq.nearCacheConfiguration(), topVer, - startReq.disabledAfterStart() + changeReq.disabledAfterStart() ); + }) + .collect(Collectors.toList()); + + Set startedCaches = startCacheInfos.stream() + .map(info -> info.getCacheDescriptor().cacheName()) + .collect(Collectors.toSet()); + + try { + cctx.cache().prepareStartCaches(startCacheInfos); + } + catch (IgniteCheckedException e) { + cctx.cache().closeCaches(startedCaches, false); + + cctx.cache().completeClientCacheChangeFuture(msg.requestId(), e); + + return null; + } + + for (StartCacheInfo startCacheInfo : startCacheInfos) { + try { + DynamicCacheDescriptor desc = startCacheInfo.getCacheDescriptor(); + + startedCaches.add(desc.cacheName()); - startedInfos.put(desc.cacheId(), startReq.nearCacheConfiguration() != null); + startedInfos.put(desc.cacheId(), startCacheInfo.getReqNearCfg() != null); CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId()); @@ -860,6 +879,8 @@ private void processCacheStartRequests( long time = System.currentTimeMillis(); + Map startCacheInfos = new LinkedHashMap<>(); + for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) { DynamicCacheDescriptor cacheDesc = action.descriptor(); @@ -895,29 +916,41 @@ private void processCacheStartRequests( } } - try { - if (startCache) { - cctx.cache().prepareCacheStart( + if (startCache) { + startCacheInfos.put( + new StartCacheInfo( req.startCacheConfiguration(), cacheDesc, nearCfg, evts.topologyVersion(), req.disabledAfterStart() - ); - - if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) { - if (fut.events().discoveryCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty()) - U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName()); - } - } + ), + req + ); } - catch (IgniteCheckedException e) { - U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " + - "[cacheName=" + req.cacheName() + ']', e); + } + + Map failedCaches = cctx.cache().prepareStartCachesIfPossible(startCacheInfos.keySet()); - cctx.cache().closeCaches(Collections.singleton(req.cacheName()), false); + failedCaches.forEach((cacheInfo, exception) -> { + U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " + + "[cacheName=" + cacheInfo.getStartedConfiguration().getName() + ']', exception); - cctx.cache().completeCacheStartFuture(req, false, e); + cctx.cache().closeCaches(Collections.singleton(cacheInfo.getStartedConfiguration().getName()), false); + + cctx.cache().completeCacheStartFuture(startCacheInfos.get(cacheInfo), false, exception); + }); + + Set failedCacheInfos = failedCaches.keySet(); + + List cacheInfos = startCacheInfos.keySet().stream() + .filter(failedCacheInfos::contains) + .collect(Collectors.toList()); + + for (StartCacheInfo info : cacheInfos) { + if (fut.cacheAddedOnExchange(info.getCacheDescriptor().cacheId(), info.getCacheDescriptor().receivedFrom())) { + if (fut.events().discoveryCache().cacheGroupAffinityNodes(info.getCacheDescriptor().groupId()).isEmpty()) + U.quietAndWarn(log, "No server nodes found for cache client: " + info.getCacheDescriptor().cacheName()); } } @@ -952,22 +985,20 @@ private void initAffinityOnCacheGroupsStart( U.doInParallel( cctx.kernalContext().getSystemExecutorService(), startedGroups, - new IgniteInClosureX() { - @Override public void applyx(CacheGroupDescriptor grpDesc) throws IgniteCheckedException { - if (crd) - initStartedGroupOnCoordinator(fut, grpDesc); - else { - CacheGroupContext grp = cctx.cache().cacheGroup(grpDesc.groupId()); + grpDesc -> { + if (crd) + initStartedGroupOnCoordinator(fut, grpDesc); + else { + CacheGroupContext grp = cctx.cache().cacheGroup(grpDesc.groupId()); - if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) { - assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion(); + if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) { + assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion(); - initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut); - } + initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut); } } - }, - null); + } + ); } /** @@ -1228,7 +1259,7 @@ private void forAllRegisteredCacheGroups(IgniteInClosureX .collect(Collectors.toList()); try { - U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null); + U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c::applyx); } catch (IgniteCheckedException e) { throw new IgniteException("Failed to execute affinity operation on cache groups", e); @@ -1255,7 +1286,7 @@ private void forAllCacheGroups(boolean crd, IgniteInClosureX idxClsHandlers0 = msgHandlers.idxClsHandlers; - IgniteBiInClosure[] cacheClsHandlers = idxClsHandlers0.get(hndId); + IgniteBiInClosure[] cacheClsHandlers = idxClsHandlers0.compute(hndId, (key, clsHandlers) -> { + if (clsHandlers == null) + clsHandlers = new IgniteBiInClosure[GridCacheMessage.MAX_CACHE_MSG_LOOKUP_INDEX]; - if (cacheClsHandlers == null) { - cacheClsHandlers = new IgniteBiInClosure[GridCacheMessage.MAX_CACHE_MSG_LOOKUP_INDEX]; + if(clsHandlers[msgIdx] != null) + return null; - idxClsHandlers0.put(hndId, cacheClsHandlers); - } + clsHandlers[msgIdx] = c; + + return clsHandlers; + }); - if (cacheClsHandlers[msgIdx] != null) + if (cacheClsHandlers == null) throw new IgniteException("Duplicate cache message ID found [hndId=" + hndId + ", type=" + type + ']'); - cacheClsHandlers[msgIdx] = c; - - msgHandlers.idxClsHandlers = idxClsHandlers0; - return; } else { @@ -1572,7 +1572,7 @@ else if (msg instanceof GridCacheGroupIdMessage) */ static class MessageHandlers { /** Indexed class handlers. */ - volatile Map idxClsHandlers = new HashMap<>(); + volatile Map idxClsHandlers = new ConcurrentHashMap<>(); /** Handler registry. */ ConcurrentMap> diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 0b8dd75b7dcf0..d47d7701d7df4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -109,6 +109,7 @@ import org.apache.ignite.internal.util.typedef.CI2; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -2619,7 +2620,8 @@ void dumpExchangeDebugInfo() { err = e; } catch (Throwable e) { - err = e; + if (!(stop && X.hasCause(e, IgniteInterruptedCheckedException.class))) + err = e; } finally { if (err == null && !stop && !reconnectNeeded) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 4a6bed4f7a51e..59703c06a6412 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache; +import javax.management.MBeanServer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -68,6 +69,7 @@ import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException; import org.apache.ignite.internal.IgniteComponentType; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.IgniteNodeAttributes; import org.apache.ignite.internal.IgniteTransactionsEx; import org.apache.ignite.internal.binary.BinaryContext; @@ -136,15 +138,16 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions; import org.apache.ignite.internal.util.F0; +import org.apache.ignite.internal.util.InitializationProtector; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.GridPlainClosure; import org.apache.ignite.internal.util.lang.IgniteOutClosureX; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CIX1; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -193,6 +196,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isPersistentCache; +import static org.apache.ignite.internal.util.IgniteUtils.doInParallel; /** * Cache processor. @@ -214,6 +218,10 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor private final boolean walFsyncWithDedicatedWorker = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER, false); + /** Enables start caches in parallel. */ + private final boolean IGNITE_ALLOW_START_CACHES_IN_PARALLEL = + IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, true); + /** Shared cache context. */ private GridCacheSharedContext sharedCtx; @@ -266,6 +274,9 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor /** MBean group for cache group metrics */ private final String CACHE_GRP_METRICS_MBEAN_GRP = "Cache groups"; + /** Protector of initialization of specific value. */ + private final InitializationProtector initializationProtector = new InitializationProtector(); + /** * @param ctx Kernal context. */ @@ -1284,71 +1295,6 @@ private void stopCacheOnReconnect(GridCacheContext cctx, List return null; } - /** - * @param cache Cache to start. - * @param schema Cache schema. - * @throws IgniteCheckedException If failed to start cache. - */ - @SuppressWarnings({"TypeMayBeWeakened", "unchecked"}) - private void startCache(GridCacheAdapter cache, QuerySchema schema) throws IgniteCheckedException { - GridCacheContext cacheCtx = cache.context(); - - CacheConfiguration cfg = cacheCtx.config(); - - // Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set. - if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY - && !(ctx.config().getMarshaller() instanceof BinaryMarshaller)) - U.warn(log, "CacheConfiguration.isStoreKeepBinary() configuration property will be ignored because " + - "BinaryMarshaller is not used"); - - // Start managers. - for (GridCacheManager mgr : F.view(cacheCtx.managers(), F.notContains(dhtExcludes(cacheCtx)))) - mgr.start(cacheCtx); - - cacheCtx.initConflictResolver(); - - if (cfg.getCacheMode() != LOCAL && GridCacheUtils.isNearEnabled(cfg)) { - GridCacheContext dhtCtx = cacheCtx.near().dht().context(); - - // Start DHT managers. - for (GridCacheManager mgr : dhtManagers(dhtCtx)) - mgr.start(dhtCtx); - - dhtCtx.initConflictResolver(); - - // Start DHT cache. - dhtCtx.cache().start(); - - if (log.isDebugEnabled()) - log.debug("Started DHT cache: " + dhtCtx.cache().name()); - } - - ctx.continuous().onCacheStart(cacheCtx); - - cacheCtx.cache().start(); - - ctx.query().onCacheStart(cacheCtx, schema); - - cacheCtx.onStarted(); - - String memPlcName = cfg.getDataRegionName(); - - if (memPlcName == null && ctx.config().getDataStorageConfiguration() != null) - memPlcName = ctx.config().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName(); - - if (log.isInfoEnabled()) { - log.info("Started cache [name=" + cfg.getName() + - ", id=" + cacheCtx.cacheId() + - (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") + - ", memoryPolicyName=" + memPlcName + - ", mode=" + cfg.getCacheMode() + - ", atomicity=" + cfg.getAtomicityMode() + - ", backups=" + cfg.getBackups() + - ", mvcc=" + cacheCtx.mvccEnabled() +']' + - ", encryptionEnabled=" + cfg.isEncryptionEnabled() +']'); - } - } - /** * @param cache Cache to stop. * @param cancel Cancel flag. @@ -1600,7 +1546,13 @@ private GridCacheContext createCache(CacheConfiguration cfg, GridCacheDrManager drMgr = pluginMgr.createComponent(GridCacheDrManager.class); CacheStoreManager storeMgr = pluginMgr.createComponent(CacheStoreManager.class); - storeMgr.initialize(cfgStore, sesHolders); + if (cfgStore == null) + storeMgr.initialize(cfgStore, sesHolders); + else + initializationProtector.protect( + cfgStore, + () -> storeMgr.initialize(cfgStore, sesHolders) + ); GridCacheContext cacheCtx = new GridCacheContext( ctx, @@ -2017,18 +1969,13 @@ public IgniteInternalFuture startCachesOnLocalJoin( IgniteInternalFuture res = sharedCtx.affinity().initCachesOnLocalJoin( locJoinCtx.cacheGroupDescriptors(), locJoinCtx.cacheDescriptors()); - for (T2 t : locJoinCtx.caches()) { - DynamicCacheDescriptor desc = t.get1(); + List startCacheInfos = locJoinCtx.caches().stream() + .map(cacheInfo -> new StartCacheInfo(cacheInfo.get1(), cacheInfo.get2(), exchTopVer, false)) + .collect(Collectors.toList()); - prepareCacheStart( - desc.cacheConfiguration(), - desc, - t.get2(), - exchTopVer, - false); + prepareStartCaches(startCacheInfos); - context().exchange().exchangerUpdateHeartbeat(); - } + context().exchange().exchangerUpdateHeartbeat(); if (log.isInfoEnabled()) log.info("Starting caches on local join performed in " + (System.currentTimeMillis() - time) + " ms."); @@ -2054,24 +2001,164 @@ boolean hasCachesReceivedFromJoin(ClusterNode node) { */ public Collection startReceivedCaches(UUID nodeId, AffinityTopologyVersion exchTopVer) throws IgniteCheckedException { - List started = cachesInfo.cachesReceivedFromJoin(nodeId); + List receivedCaches = cachesInfo.cachesReceivedFromJoin(nodeId); - for (DynamicCacheDescriptor desc : started) { - IgnitePredicate filter = desc.groupDescriptor().config().getNodeFilter(); + List startCacheInfos = receivedCaches.stream() + .filter(desc -> isLocalAffinity(desc.groupDescriptor().config())) + .map(desc -> new StartCacheInfo(desc, null, exchTopVer, false)) + .collect(Collectors.toList()); - if (CU.affinityNode(ctx.discovery().localNode(), filter)) { - prepareCacheStart( - desc.cacheConfiguration(), - desc, - null, - exchTopVer, - false); + prepareStartCaches(startCacheInfos); + + return receivedCaches; + } + + /** + * @param cacheConfiguration Checked configuration. + * @return {@code true} if local node is affinity node for cache. + */ + private boolean isLocalAffinity(CacheConfiguration cacheConfiguration) { + return CU.affinityNode(ctx.discovery().localNode(), cacheConfiguration.getNodeFilter()); + } + + /** + * Start all input caches in parallel. + * + * @param startCacheInfos All caches information for start. + */ + void prepareStartCaches(Collection startCacheInfos) throws IgniteCheckedException { + prepareStartCaches(startCacheInfos, (data, operation) -> { + operation.accept(data);// PROXY + }); + } + + /** + * Trying to start all input caches in parallel and skip failed caches. + * + * @param startCacheInfos Caches info for start. + * @return Caches which was failed. + * @throws IgniteCheckedException if failed. + */ + Map prepareStartCachesIfPossible(Collection startCacheInfos) throws IgniteCheckedException { + HashMap failedCaches = new HashMap<>(); + + prepareStartCaches(startCacheInfos, (data, operation) -> { + try { + operation.accept(data); + } + catch (IgniteInterruptedCheckedException e) { + throw e; + } + catch (IgniteCheckedException e) { + log.warning("Cache can not be started : cache=" + data.getStartedConfiguration().getName()); + + failedCaches.put(data, e); + } + }); + + return failedCaches; + } + + /** + * Start all input caches in parallel. + * + * @param startCacheInfos All caches information for start. + * @param cacheStartFailHandler Fail handler for one cache start. + */ + private void prepareStartCaches( + Collection startCacheInfos, + StartCacheFailHandler cacheStartFailHandler + ) throws IgniteCheckedException { + if (!IGNITE_ALLOW_START_CACHES_IN_PARALLEL || startCacheInfos.size() <= 1) { + for (StartCacheInfo startCacheInfo : startCacheInfos) { + cacheStartFailHandler.handle( + startCacheInfo, + cacheInfo -> prepareCacheStart( + cacheInfo.getCacheDescriptor().cacheConfiguration(), + cacheInfo.getCacheDescriptor(), + cacheInfo.getReqNearCfg(), + cacheInfo.getExchangeTopVer(), + cacheInfo.isDisabledAfterStart() + ) + ); context().exchange().exchangerUpdateHeartbeat(); } } + else { + Map cacheContexts = new ConcurrentHashMap<>(); + + int parallelismLvl = sharedCtx.kernalContext().config().getSystemThreadPoolSize(); + + // Reserve at least 2 threads for system operations. + parallelismLvl = Math.max(1, parallelismLvl - 2); + + doInParallel( + parallelismLvl, + sharedCtx.kernalContext().getSystemExecutorService(), + startCacheInfos, + startCacheInfo -> + cacheStartFailHandler.handle( + startCacheInfo, + cacheInfo -> { + GridCacheContext cacheCtx = prepareCacheContext( + cacheInfo.getCacheDescriptor().cacheConfiguration(), + cacheInfo.getCacheDescriptor(), + cacheInfo.getReqNearCfg(), + cacheInfo.getExchangeTopVer(), + cacheInfo.isDisabledAfterStart() + ); + cacheContexts.put(cacheInfo, cacheCtx); + + context().exchange().exchangerUpdateHeartbeat(); + } + ) + ); - return started; + /* + * This hack required because we can't start sql schema in parallel by folowing reasons: + * * checking index to duplicate(and other checking) require one order on every nodes. + * * onCacheStart and createSchema contains a lot of mutex. + * + * TODO IGNITE-9729 + */ + Set successfullyPreparedCaches = cacheContexts.keySet(); + + List cacheInfosInOriginalOrder = startCacheInfos.stream() + .filter(successfullyPreparedCaches::contains) + .collect(Collectors.toList()); + + for (StartCacheInfo startCacheInfo : cacheInfosInOriginalOrder) { + cacheStartFailHandler.handle( + startCacheInfo, + cacheInfo -> { + ctx.query().onCacheStart( + cacheContexts.get(cacheInfo), + cacheInfo.getCacheDescriptor().schema() != null + ? cacheInfo.getCacheDescriptor().schema() + : new QuerySchema() + ); + + context().exchange().exchangerUpdateHeartbeat(); + } + ); + } + + doInParallel( + parallelismLvl, + sharedCtx.kernalContext().getSystemExecutorService(), + cacheContexts.entrySet(), + cacheCtxEntry -> + cacheStartFailHandler.handle( + cacheCtxEntry.getKey(), + cacheInfo -> { + onCacheStarted(cacheCtxEntry.getValue()); + + context().exchange().exchangerUpdateHeartbeat(); + } + ) + ); + } } /** @@ -2089,6 +2176,32 @@ void prepareCacheStart( @Nullable NearCacheConfiguration reqNearCfg, AffinityTopologyVersion exchTopVer, boolean disabledAfterStart + ) throws IgniteCheckedException { + GridCacheContext cacheCtx = prepareCacheContext(startCfg, desc, reqNearCfg, exchTopVer, disabledAfterStart); + + ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); + + onCacheStarted(cacheCtx); + } + + /** + * Preparing cache context to start. + * + * @param startCfg Cache configuration to use. + * @param desc Cache descriptor. + * @param reqNearCfg Near configuration if specified for client cache start request. + * @param exchTopVer Current exchange version. + * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will change + * state of proxies to restarting + * @return Created {@link GridCacheContext}. + * @throws IgniteCheckedException if failed. + */ + private GridCacheContext prepareCacheContext( + CacheConfiguration startCfg, + DynamicCacheDescriptor desc, + @Nullable NearCacheConfiguration reqNearCfg, + AffinityTopologyVersion exchTopVer, + boolean disabledAfterStart ) throws IgniteCheckedException { assert !caches.containsKey(startCfg.getName()) : startCfg.getName(); @@ -2096,65 +2209,124 @@ void prepareCacheStart( CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(ccfg); - boolean affNode; + boolean affNode = checkForAffinityNode(desc, reqNearCfg, ccfg); - if (ccfg.getCacheMode() == LOCAL) { - affNode = true; + preparePageStore(desc, affNode); + + CacheGroupContext grp = prepareCacheGroup(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName()); + + GridCacheContext cacheCtx = createCache(ccfg, + grp, + null, + desc, + exchTopVer, + cacheObjCtx, + affNode, + true, + disabledAfterStart + ); + initCacheContext(cacheCtx, ccfg, desc.deploymentId()); + + return cacheCtx; + } + + /** + * Check for affinity node and customize near configuration if needed. + * + * @param desc Cache descriptor. + * @param reqNearCfg Near configuration if specified for client cache start request. + * @param ccfg Cache configuration to use. + * @return {@code true} if it is affinity node for cache. + */ + private boolean checkForAffinityNode( + DynamicCacheDescriptor desc, + @Nullable NearCacheConfiguration reqNearCfg, + CacheConfiguration ccfg + ) { + if (ccfg.getCacheMode() == LOCAL) { ccfg.setNearConfiguration(null); - } - else if (CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().config().getNodeFilter())) - affNode = true; - else { - affNode = false; - ccfg.setNearConfiguration(reqNearCfg); + return true; } - if (sharedCtx.pageStore() != null && affNode) - sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), desc.toStoredData()); - - String grpName = startCfg.getGroupName(); + if (isLocalAffinity(desc.groupDescriptor().config())) + return true; - CacheGroupContext grp = null; + ccfg.setNearConfiguration(reqNearCfg); - if (grpName != null) { - for (CacheGroupContext grp0 : cacheGrps.values()) { - if (grp0.sharedGroup() && grpName.equals(grp0.name())) { - grp = grp0; + return false; + } - break; - } - } + /** + * Prepare page store for start cache. + * + * @param desc Cache descriptor. + * @param affNode {@code true} if it is affinity node for cache. + * @throws IgniteCheckedException if failed. + */ + private void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException { + if (sharedCtx.pageStore() != null && affNode) + initializationProtector.protect( + desc.groupDescriptor().groupId(), + () -> sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), desc.toStoredData()) + ); + } - if (grp == null) { - grp = startCacheGroup(desc.groupDescriptor(), + /** + * Prepare cache group to start cache. + * + * @param desc Cache descriptor. + * @param exchTopVer Current exchange version. + * @param cacheObjCtx Cache object context. + * @param affNode {@code true} if it is affinity node for cache. + * @param grpName Group name. + * @return Prepared cache group context. + * @throws IgniteCheckedException if failed. + */ + private CacheGroupContext prepareCacheGroup( + DynamicCacheDescriptor desc, + AffinityTopologyVersion exchTopVer, + CacheObjectContext cacheObjCtx, + boolean affNode, + String grpName + ) throws IgniteCheckedException { + if (grpName != null) { + return initializationProtector.protect( + desc.groupId(), + () -> findCacheGroup(grpName), + () -> startCacheGroup( + desc.groupDescriptor(), desc.cacheType(), affNode, cacheObjCtx, - exchTopVer); - } - } - else { - grp = startCacheGroup(desc.groupDescriptor(), - desc.cacheType(), - affNode, - cacheObjCtx, - exchTopVer); + exchTopVer + ) + ); } - GridCacheContext cacheCtx = createCache(ccfg, - grp, - null, - desc, - exchTopVer, - cacheObjCtx, + return startCacheGroup(desc.groupDescriptor(), + desc.cacheType(), affNode, - true, - disabledAfterStart + cacheObjCtx, + exchTopVer ); + } - cacheCtx.dynamicDeploymentId(desc.deploymentId()); + /** + * Initialize created cache context. + * + * @param cacheCtx Cache context to initializtion. + * @param cfg Cache configuration. + * @param deploymentId Dynamic deployment ID. + * @throws IgniteCheckedException if failed. + */ + private void initCacheContext( + GridCacheContext cacheCtx, + CacheConfiguration cfg, + IgniteUuid deploymentId + ) throws IgniteCheckedException { + cacheCtx.dynamicDeploymentId(deploymentId); GridCacheAdapter cache = cacheCtx.cache(); @@ -2162,13 +2334,84 @@ else if (CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().con caches.put(cacheCtx.name(), cache); - startCache(cache, desc.schema() != null ? desc.schema() : new QuerySchema()); + // Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set. + if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY + && !(ctx.config().getMarshaller() instanceof BinaryMarshaller)) + U.warn(log, "CacheConfiguration.isStoreKeepBinary() configuration property will be ignored because " + + "BinaryMarshaller is not used"); + + // Start managers. + for (GridCacheManager mgr : F.view(cacheCtx.managers(), F.notContains(dhtExcludes(cacheCtx)))) + mgr.start(cacheCtx); + + cacheCtx.initConflictResolver(); + + if (cfg.getCacheMode() != LOCAL && GridCacheUtils.isNearEnabled(cfg)) { + GridCacheContext dhtCtx = cacheCtx.near().dht().context(); + + // Start DHT managers. + for (GridCacheManager mgr : dhtManagers(dhtCtx)) + mgr.start(dhtCtx); + + dhtCtx.initConflictResolver(); + + // Start DHT cache. + dhtCtx.cache().start(); + + if (log.isDebugEnabled()) + log.debug("Started DHT cache: " + dhtCtx.cache().name()); + } + + ctx.continuous().onCacheStart(cacheCtx); + + cacheCtx.cache().start(); + } + + /** + * Handle of cache context which was fully prepared. + * + * @param cacheCtx Fully prepared context. + * @throws IgniteCheckedException if failed. + */ + private void onCacheStarted(GridCacheContext cacheCtx) throws IgniteCheckedException { + GridCacheAdapter cache = cacheCtx.cache(); + CacheConfiguration cfg = cacheCtx.config(); + CacheGroupContext grp = cacheGrps.get(cacheCtx.groupId()); + + cacheCtx.onStarted(); + + String dataRegion = cfg.getDataRegionName(); + + if (dataRegion == null && ctx.config().getDataStorageConfiguration() != null) + dataRegion = ctx.config().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName(); + + if (log.isInfoEnabled()) { + log.info("Started cache [name=" + cfg.getName() + + ", id=" + cacheCtx.cacheId() + + (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") + + ", dataRegionName=" + dataRegion + + ", mode=" + cfg.getCacheMode() + + ", atomicity=" + cfg.getAtomicityMode() + + ", backups=" + cfg.getBackups() + + ", mvcc=" + cacheCtx.mvccEnabled() + ']'); + } grp.onCacheStarted(cacheCtx); onKernalStart(cache); } + /** + * @param grpName Group name. + * @return Found group or null. + */ + private CacheGroupContext findCacheGroup(String grpName) { + return cacheGrps.values().stream() + .filter(grp -> grp.sharedGroup() && grpName.equals(grp.name())) + .findAny() + .orElse(null); + } + /** * Restarts proxies of caches if they was marked as restarting. Requires external synchronization - shouldn't be * called concurrently with another caches restart. @@ -4839,7 +5082,7 @@ private DynamicCacheChangeRequest prepareCacheChangeRequest( // Check if we were asked to start a near cache. if (nearCfg != null) { - if (CU.affinityNode(ctx.discovery().localNode(), descCfg.getNodeFilter())) { + if (isLocalAffinity(descCfg)) { // If we are on a data node and near cache was enabled, return success, else - fail. if (descCfg.getNearConfiguration() != null) return null; @@ -4851,7 +5094,7 @@ private DynamicCacheChangeRequest prepareCacheChangeRequest( // If local node has near cache, return success. req.clientStartOnly(true); } - else if (!CU.affinityNode(ctx.discovery().localNode(), descCfg.getNodeFilter())) + else if (!isLocalAffinity(descCfg)) req.clientStartOnly(true); req.deploymentId(desc.deploymentId()); @@ -5019,6 +5262,22 @@ public T clone(final T obj) throws IgniteCheckedException { }); } + /** + * Handle of fail during cache start. + * + * @param Type of started data. + */ + private static interface StartCacheFailHandler { + /** + * Handle of fail. + * + * @param data Start data. + * @param startCacheOperation Operation for start cache. + * @throws IgniteCheckedException if failed. + */ + void handle(T data, IgniteThrowableConsumer startCacheOperation) throws IgniteCheckedException; + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java new file mode 100644 index 0000000000000..a5aea26453ff6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.jetbrains.annotations.Nullable; + +/** + * Specific cache information for start. + */ +public class StartCacheInfo { + /** Cache configuration for start. */ + private final CacheConfiguration startedConf; + + /** Cache descriptor for start. */ + private final DynamicCacheDescriptor desc; + + /** Near cache configuration for start. */ + private final @Nullable NearCacheConfiguration reqNearCfg; + + /** Exchange topology version in which starting happened. */ + private final AffinityTopologyVersion exchTopVer; + + /** Disable started cache after start or not. */ + private final boolean disabledAfterStart; + + /** + * @param desc Cache configuration for start. + * @param reqNearCfg Near cache configuration for start. + * @param exchTopVer Exchange topology version in which starting happened. + * @param disabledAfterStart Disable started cache after start or not. + */ + public StartCacheInfo(DynamicCacheDescriptor desc, + NearCacheConfiguration reqNearCfg, + AffinityTopologyVersion exchTopVer, boolean disabledAfterStart) { + this(desc.cacheConfiguration(), desc, reqNearCfg, exchTopVer, disabledAfterStart); + } + + /** + * @param conf Cache configuration for start. + * @param desc Cache descriptor for start. + * @param reqNearCfg Near cache configuration for start. + * @param exchTopVer Exchange topology version in which starting happened. + * @param disabledAfterStart Disable started cache after start or not. + */ + public StartCacheInfo(CacheConfiguration conf, DynamicCacheDescriptor desc, + NearCacheConfiguration reqNearCfg, + AffinityTopologyVersion exchTopVer, boolean disabledAfterStart) { + startedConf = conf; + this.desc = desc; + this.reqNearCfg = reqNearCfg; + this.exchTopVer = exchTopVer; + this.disabledAfterStart = disabledAfterStart; + } + + /** + * @return Cache configuration for start. + */ + public CacheConfiguration getStartedConfiguration() { + return startedConf; + } + + /** + * @return Cache descriptor for start. + */ + public DynamicCacheDescriptor getCacheDescriptor() { + return desc; + } + + /** + * @return Near cache configuration for start. + */ + @Nullable public NearCacheConfiguration getReqNearCfg() { + return reqNearCfg; + } + + /** + * @return Exchange topology version in which starting happened. + */ + public AffinityTopologyVersion getExchangeTopVer() { + return exchTopVer; + } + + /** + * @return Disable started cache after start or not. + */ + public boolean isDisabledAfterStart() { + return disabledAfterStart; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(StartCacheInfo.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index d4948576737f6..0c2cbe26c54f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3512,36 +3512,34 @@ private void validatePartitionsState() { U.doInParallel( cctx.kernalContext().getSystemExecutorService(), nonLocalCacheGroupDescriptors(), - new IgniteInClosureX() { - @Override public void applyx(CacheGroupDescriptor grpDesc) { - CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId()); - - GridDhtPartitionTopology top = grpCtx != null - ? grpCtx.topology() - : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache()); - - // Do not validate read or write through caches or caches with disabled rebalance - // or ExpiryPolicy is set or validation is disabled. - if (grpCtx == null - || grpCtx.config().isReadThrough() - || grpCtx.config().isWriteThrough() - || grpCtx.config().getCacheStoreFactory() != null - || grpCtx.config().getRebalanceDelay() == -1 - || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE - || grpCtx.config().getExpiryPolicyFactory() == null - || SKIP_PARTITION_SIZE_VALIDATION) - return; + grpDesc -> { + CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId()); + + GridDhtPartitionTopology top = grpCtx != null + ? grpCtx.topology() + : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache()); + + // Do not validate read or write through caches or caches with disabled rebalance + // or ExpiryPolicy is set or validation is disabled. + if (grpCtx == null + || grpCtx.config().isReadThrough() + || grpCtx.config().isWriteThrough() + || grpCtx.config().getCacheStoreFactory() != null + || grpCtx.config().getRebalanceDelay() == -1 + || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE + || grpCtx.config().getExpiryPolicyFactory() == null + || SKIP_PARTITION_SIZE_VALIDATION) + return; - try { - validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs); - } - catch (IgniteCheckedException ex) { - log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage()); - // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833 - } + try { + validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs); } - }, - null); + catch (IgniteCheckedException ex) { + log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage()); + // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833 + } + } + ); } catch (IgniteCheckedException e) { throw new IgniteException("Failed to validate partitions state", e); @@ -3561,21 +3559,19 @@ private void assignPartitionsStates() { U.doInParallel( cctx.kernalContext().getSystemExecutorService(), nonLocalCacheGroupDescriptors(), - new IgniteInClosureX() { - @Override public void applyx(CacheGroupDescriptor grpDesc) { - CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId()); + grpDesc -> { + CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId()); - GridDhtPartitionTopology top = grpCtx != null - ? grpCtx.topology() - : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache()); + GridDhtPartitionTopology top = grpCtx != null + ? grpCtx.topology() + : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache()); - if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration())) - assignPartitionSizes(top); - else - assignPartitionStates(top); - } - }, - null); + if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration())) + assignPartitionSizes(top); + else + assignPartitionStates(top); + } + ); } catch (IgniteCheckedException e) { throw new IgniteException("Failed to assign partition states", e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index d10a7c7c9cf58..6dba7b885b14c 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -142,6 +142,7 @@ import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; @@ -216,19 +217,16 @@ import org.apache.ignite.internal.util.lang.GridClosureException; import org.apache.ignite.internal.util.lang.GridPeerDeployAware; import org.apache.ignite.internal.util.lang.GridTuple; -import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.typedef.C1; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.P1; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; -import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.lang.IgniteFutureCancelledException; @@ -236,6 +234,7 @@ import org.apache.ignite.lang.IgniteOutClosure; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteProductVersion; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.lifecycle.LifecycleAware; import org.apache.ignite.marshaller.Marshaller; @@ -10747,54 +10746,91 @@ public static ClusterNode randomServerNode(GridKernalContext ctx) { } /** + * Execute operation on data in parallel. + * * @param executorSvc Service for parallel execution. * @param srcDatas List of data for parallelization. - * @param consumer Logic for execution of on each item of data. - * @param errHnd Optionan error handler. If not {@code null}, an error of each item execution will be passed to - * this handler. If error handler is not {@code null}, the exception will not be thrown from this method. + * @param operation Logic for execution of on each item of data. * @param Type of data. - * @return List of (item, execution future) tuples. - * @throws IgniteCheckedException If parallel execution failed and {@code errHnd} is {@code null}. + * @throws IgniteCheckedException if parallel execution was failed. */ - public static List>> doInParallel( + public static void doInParallel(ExecutorService executorSvc, Collection srcDatas, + IgniteThrowableConsumer operation) throws IgniteCheckedException, IgniteInterruptedCheckedException { + doInParallel(srcDatas.size(), executorSvc, srcDatas, operation); + } + + /** + * Execute operation on data in parallel. + * + * @param parallelismLvl Number of threads on which it should be executed. + * @param executorSvc Service for parallel execution. + * @param srcDatas List of data for parallelization. + * @param operation Logic for execution of on each item of data. + * @param Type of data. + * @throws IgniteCheckedException if parallel execution was failed. + */ + public static void doInParallel( + int parallelismLvl, ExecutorService executorSvc, Collection srcDatas, - IgniteInClosureX consumer, - @Nullable IgniteBiInClosure errHnd - ) throws IgniteCheckedException { - List>> consumerFutures = srcDatas.stream() - .map(item -> new T2<>( - item, - executorSvc.submit(() -> { - consumer.apply(item); + IgniteThrowableConsumer operation + ) throws IgniteCheckedException, IgniteInterruptedCheckedException { + List> batches = IntStream.range(0, parallelismLvl) + .mapToObj(i -> new ArrayList()) + .collect(Collectors.toList()); - return null; - }))) + int i = 0; + + for (T src : srcDatas) + batches.get(i++ % parallelismLvl).add(src); + + List> consumerFutures = batches.stream() + .filter(batch -> !batch.isEmpty()) + .map(batch -> executorSvc.submit(() -> { + for (T item : batch) + operation.accept(item); + + return null; + })) .collect(Collectors.toList()); - IgniteCheckedException composite = null; + Throwable error =null; - for (T2> tup : consumerFutures) { + for (Future future : consumerFutures) { try { - getUninterruptibly(tup.get2()); + future.get(); } - catch (ExecutionException e) { - if (errHnd != null) - errHnd.apply(tup.get1(), e.getCause()); - else { - if (composite == null) - composite = new IgniteCheckedException("Failed to execute one of the tasks " + - "(see suppressed exception for details)"); + catch (InterruptedException e) { + Thread.currentThread().interrupt(); - composite.addSuppressed(e.getCause()); - } + throw new IgniteInterruptedCheckedException(e); + } + catch (ExecutionException e) { + if(error == null) + error = e.getCause(); + else + error.addSuppressed(e.getCause()); + } + catch (CancellationException e) { + if(error == null) + error = e; + else + error.addSuppressed(e); } } - if (composite != null) - throw composite; + if (error != null) { + if (error instanceof IgniteCheckedException) + throw (IgniteCheckedException)error; - return consumerFutures; + if (error instanceof RuntimeException) + throw (RuntimeException)error; + + if (error instanceof Error) + throw (Error)error; + + throw new IgniteCheckedException(error); + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java b/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java new file mode 100644 index 0000000000000..7c501c42486c9 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.util; + +import java.util.concurrent.locks.Lock; +import java.util.function.Supplier; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.util.lang.IgniteThrowableRunner; + +/** + * Class for avoid multiple initialization of specific value from various threads. + */ +public class InitializationProtector { + /** Default striped lock concurrency level. */ + private static final int DEFAULT_CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors(); + + /** Striped lock. */ + private GridStripedLock stripedLock = new GridStripedLock(DEFAULT_CONCURRENCY_LEVEL); + + /** + * @param protectedKey Unique value by which initialization code should be run only one time. + * @param initializedVal Supplier for given already initialized value if it exist or null as sign that + * initialization required. + * @param initializationCode Code for initialization value corresponding protectedKey. Should be idempotent. + * @param Type of initialization value. + * @return Initialized value. + * @throws IgniteCheckedException if initialization was failed. + */ + public T protect(Object protectedKey, Supplier initializedVal, + IgniteThrowableRunner initializationCode) throws IgniteCheckedException { + T value = initializedVal.get(); + + if (value != null) + return value; + + Lock lock = stripedLock.getLock(protectedKey.hashCode() % stripedLock.concurrencyLevel()); + + lock.lock(); + try { + value = initializedVal.get(); + + if (value != null) + return value; + + initializationCode.run(); + + return initializedVal.get(); + } + finally { + lock.unlock(); + } + } + + /** + * It method allows to avoid simultaneous initialization from various threads. + * + * @param protectedKey Unique value by which initialization code should be run only from one thread in one time. + * @param initializationCode Code for initialization value corresponding protectedKey. Should be idempotent. + * @throws IgniteCheckedException if initialization was failed. + */ + public void protect(Object protectedKey, IgniteThrowableRunner initializationCode) throws IgniteCheckedException { + protect(protectedKey, () -> null, initializationCode); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java new file mode 100644 index 0000000000000..46813a9a4cf29 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.util.lang; + +import java.io.Serializable; +import org.apache.ignite.IgniteCheckedException; + +/** + * Represents an operation that accepts a single input argument and returns no result. Unlike most other functional + * interfaces, {@code IgniteThrowableConsumer} is expected to operate via side-effects. + * + * @param Type of closure parameter. + */ +public interface IgniteThrowableConsumer extends Serializable { + /** + * Consumer body. + * + * @param e Consumer parameter. + * @throws IgniteCheckedException if body execution was failed. + */ + public void accept(E e) throws IgniteCheckedException; +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java new file mode 100644 index 0000000000000..a5c95e1a3e575 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.util.lang; + +import org.apache.ignite.IgniteCheckedException; + +/** + * Represents a throwable runner. + */ +public interface IgniteThrowableRunner { + /** + * Execute a body. + */ + void run() throws IgniteCheckedException; +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java new file mode 100644 index 0000000000000..4e30d1cb6bab9 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.ArrayList; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * + */ +public class CacheStartInParallelTest extends GridCommonAbstractTest { + /** */ + private static final int CACHES_COUNT = 40; + + /** */ + private static final String STATIC_CACHE_PREFIX = "static-cache-"; + + /** */ + private static final String DYNAMIC_CACHE_PREFIX = "dynamic-cache-"; + + /** */ + private static boolean isStaticCache = true; + + /** */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setSystemThreadPoolSize(10); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + long sz = 100 * 1024 * 1024; + + DataStorageConfiguration memCfg = new DataStorageConfiguration().setPageSize(1024) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setPersistenceEnabled(true).setInitialSize(sz).setMaxSize(sz)) + .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(24L * 60 * 60 * 1000); + + cfg.setDataStorageConfiguration(memCfg); + + if (isStaticCache) { + ArrayList staticCaches = new ArrayList<>(CACHES_COUNT); + + for (int i = 0; i < CACHES_COUNT; i++) + staticCaches.add(cacheConfiguration(STATIC_CACHE_PREFIX + i)); + + cfg.setCacheConfiguration(staticCaches.toArray(new CacheConfiguration[CACHES_COUNT])); + } + + return cfg; + } + + /** + * @param cacheName Cache name. + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration(String cacheName) { + CacheConfiguration cfg = defaultCacheConfiguration(); + + cfg.setName(cacheName); + cfg.setBackups(1); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanupTestData(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanupTestData(); + } + + /** */ + private void cleanupTestData() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.clearProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL); + + isStaticCache = true; + } + + /** + * Checking that start static caches in parallel faster than consistenly. + * + * @throws Exception if fail. + */ + public void testParallelizationAcceleratesStartOfStaticCaches() throws Exception { + //start caches consistently. + System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "false"); + + long startTime = System.currentTimeMillis(); + + IgniteEx igniteEx = startGrid(0); + + igniteEx.cluster().active(true); + + long totalStartTimeConsistently = System.currentTimeMillis() - startTime; + + //check cache started. + for (int i = 0; i < CACHES_COUNT; i++) + igniteEx.cache(STATIC_CACHE_PREFIX + i).put(i, i); + + stopAllGrids(); + + //start caches in parallel. + System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "true"); + + startTime = System.currentTimeMillis(); + + igniteEx = startGrid(0); + + igniteEx.cluster().active(true); + + long totalStartTimeInParallel = System.currentTimeMillis() - startTime; + + for (int i = 0; i < CACHES_COUNT; i++) + igniteEx.cache(STATIC_CACHE_PREFIX + i).put(i, i); + + stopAllGrids(); + + assertTrue("Consistently cache stat time : " + totalStartTimeConsistently + + "Parallelization cache stat time : " + totalStartTimeInParallel, + totalStartTimeConsistently > totalStartTimeInParallel); + } + + /** + * Checking that start dynamic caches in parallel faster than consistenly. + * + * @throws Exception if fail. + */ + public void testParallelizationAcceleratesStartOfCaches2() throws Exception { + //prepare dynamic caches. + isStaticCache = false; + + IgniteEx igniteEx = startGrid(0); + + igniteEx.cluster().active(true); + + for (int i = 0; i < CACHES_COUNT; i++) + igniteEx.getOrCreateCache(DYNAMIC_CACHE_PREFIX + i); + + stopAllGrids(); + + //start caches consistently. + System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "false"); + + igniteEx = startGrid(0); + long startTime = System.currentTimeMillis(); + + igniteEx.cluster().active(true); + + long totalStartTimeConsistently = System.currentTimeMillis() - startTime; + + for (int i = 0; i < CACHES_COUNT; i++) + igniteEx.cache(DYNAMIC_CACHE_PREFIX + i); + + stopAllGrids(); + + //start caches in parallel. + System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "true"); + + startTime = System.currentTimeMillis(); + + igniteEx = startGrid(0); + + igniteEx.cluster().active(true); + + long totalStartTimeInParallel = System.currentTimeMillis() - startTime; + + for (int i = 0; i < CACHES_COUNT; i++) + igniteEx.cache(DYNAMIC_CACHE_PREFIX + i).put(i, i); + + stopAllGrids(); + + assertTrue("Consistently cache stat time : " + totalStartTimeConsistently + + "Parallelization cache stat time : " + totalStartTimeInParallel, + totalStartTimeConsistently > totalStartTimeInParallel); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java index 870ce675ec688..66453b80ca75a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java @@ -17,15 +17,15 @@ package org.apache.ignite.internal.processors.cache.distributed; +import javax.cache.Cache; +import javax.cache.configuration.Factory; +import javax.cache.integration.CacheLoaderException; +import javax.cache.integration.CacheWriterException; import java.util.Collection; import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; -import javax.cache.Cache; -import javax.cache.configuration.Factory; -import javax.cache.integration.CacheLoaderException; -import javax.cache.integration.CacheWriterException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.store.CacheStore; @@ -61,7 +61,7 @@ public class IgniteCrossCacheTxStoreSelfTest extends GridCommonAbstractTest { CacheConfiguration cfg3 = cacheConfiguration("cacheC", new SecondStoreFactory()); CacheConfiguration cfg4 = cacheConfiguration("cacheD", null); - cfg.setCacheConfiguration(cfg1, cfg2, cfg3, cfg4); + cfg.setCacheConfiguration(cfg4, cfg2, cfg3, cfg1); return cfg; } @@ -92,6 +92,8 @@ private CacheConfiguration cacheConfiguration(String cacheName, Factory { private Ignite ignite; /** {@inheritDoc} */ - @Override public CacheStore create() { + @Override public synchronized CacheStore create() { String igniteInstanceName = ignite.name(); - CacheStore store = firstStores.get(igniteInstanceName); - - if (store == null) - store = F.addIfAbsent(firstStores, igniteInstanceName, new TestStore()); - - return store; + return firstStores.computeIfAbsent(igniteInstanceName, (k) -> new TestStore()); } } @@ -384,12 +381,7 @@ private static class SecondStoreFactory implements Factory { @Override public CacheStore create() { String igniteInstanceName = ignite.name(); - CacheStore store = secondStores.get(igniteInstanceName); - - if (store == null) - store = F.addIfAbsent(secondStores, igniteInstanceName, new TestStore()); - - return store; + return secondStores.computeIfAbsent(igniteInstanceName, (k) -> new TestStore()); } } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java index 61a076ed65f34..13a104431ce84 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java @@ -44,6 +44,10 @@ import java.util.List; import java.util.Random; import java.util.UUID; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterGroup; @@ -876,6 +880,76 @@ public void testIsOldestNodeVersionAtLeast() { assertTrue(U.isOldestNodeVersionAtLeast(v250ts, Arrays.asList(node250, node250ts))); } + /** + * + */ + public void testDoInParallel() throws Throwable { + CyclicBarrier barrier = new CyclicBarrier(3); + + IgniteUtils.doInParallel(3, + Executors.newFixedThreadPool(3), + Arrays.asList(1, 2, 3), + i -> { + try { + barrier.await(1, TimeUnit.SECONDS); + } + catch (Exception e) { + throw new IgniteCheckedException(e); + } + } + ); + } + + /** + * + */ + public void testDoInParallelBatch() { + CyclicBarrier barrier = new CyclicBarrier(3); + + try { + IgniteUtils.doInParallel(2, + Executors.newFixedThreadPool(3), + Arrays.asList(1, 2, 3), + i -> { + try { + barrier.await(400, TimeUnit.MILLISECONDS); + } + catch (Exception e) { + throw new IgniteCheckedException(e); + } + } + ); + + fail("Should throw timeout exception"); + } + catch (Exception e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + } + + /** + * + */ + public void testDoInParallelException() { + String expectedException = "ExpectedException"; + + try { + IgniteUtils.doInParallel(3, + Executors.newFixedThreadPool(1), + Arrays.asList(1, 2, 3), + i -> { + if (i == 1) + throw new IgniteCheckedException(expectedException); + } + ); + + fail("Should throw ParallelExecutionException"); + } + catch (IgniteCheckedException e) { + assertEquals(expectedException, e.getMessage()); + } + } + /** * Test enum. */ diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java index 6c48ecc2da6f7..58941bc2f4ea3 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java @@ -35,6 +35,7 @@ import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest; import org.apache.ignite.internal.processors.cache.distributed.CachePageWriteLockUnlockTest; import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheStartInParallelTest; import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest; import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest; @@ -99,6 +100,8 @@ public static TestSuite suite(Set ignoredTests) throws Exception { suite.addTestSuite(CacheRentingStateRepairTest.class); + suite.addTestSuite(CacheStartInParallelTest.class); + suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class); suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class); From 3be6cf7781bafb75a571a32814a3626f07d672f1 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Tue, 23 Oct 2018 11:01:40 +0300 Subject: [PATCH 031/403] IGNITE-8006 Removed non-representative flaky test --- .../distributed/CacheStartInParallelTest.java | 219 ------------------ .../testsuites/IgniteCacheTestSuite7.java | 3 - 2 files changed, 222 deletions(-) delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java deleted file mode 100644 index 4e30d1cb6bab9..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.distributed; - -import java.util.ArrayList; -import org.apache.ignite.IgniteSystemProperties; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.WALMode; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -/** - * - */ -public class CacheStartInParallelTest extends GridCommonAbstractTest { - /** */ - private static final int CACHES_COUNT = 40; - - /** */ - private static final String STATIC_CACHE_PREFIX = "static-cache-"; - - /** */ - private static final String DYNAMIC_CACHE_PREFIX = "dynamic-cache-"; - - /** */ - private static boolean isStaticCache = true; - - /** */ - private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setSystemThreadPoolSize(10); - - TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); - - discoSpi.setIpFinder(IP_FINDER); - - cfg.setDiscoverySpi(discoSpi); - - long sz = 100 * 1024 * 1024; - - DataStorageConfiguration memCfg = new DataStorageConfiguration().setPageSize(1024) - .setDefaultDataRegionConfiguration( - new DataRegionConfiguration().setPersistenceEnabled(true).setInitialSize(sz).setMaxSize(sz)) - .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(24L * 60 * 60 * 1000); - - cfg.setDataStorageConfiguration(memCfg); - - if (isStaticCache) { - ArrayList staticCaches = new ArrayList<>(CACHES_COUNT); - - for (int i = 0; i < CACHES_COUNT; i++) - staticCaches.add(cacheConfiguration(STATIC_CACHE_PREFIX + i)); - - cfg.setCacheConfiguration(staticCaches.toArray(new CacheConfiguration[CACHES_COUNT])); - } - - return cfg; - } - - /** - * @param cacheName Cache name. - * @return Cache configuration. - */ - private CacheConfiguration cacheConfiguration(String cacheName) { - CacheConfiguration cfg = defaultCacheConfiguration(); - - cfg.setName(cacheName); - cfg.setBackups(1); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - cleanupTestData(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - cleanupTestData(); - } - - /** */ - private void cleanupTestData() throws Exception { - stopAllGrids(); - - cleanPersistenceDir(); - - System.clearProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL); - - isStaticCache = true; - } - - /** - * Checking that start static caches in parallel faster than consistenly. - * - * @throws Exception if fail. - */ - public void testParallelizationAcceleratesStartOfStaticCaches() throws Exception { - //start caches consistently. - System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "false"); - - long startTime = System.currentTimeMillis(); - - IgniteEx igniteEx = startGrid(0); - - igniteEx.cluster().active(true); - - long totalStartTimeConsistently = System.currentTimeMillis() - startTime; - - //check cache started. - for (int i = 0; i < CACHES_COUNT; i++) - igniteEx.cache(STATIC_CACHE_PREFIX + i).put(i, i); - - stopAllGrids(); - - //start caches in parallel. - System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "true"); - - startTime = System.currentTimeMillis(); - - igniteEx = startGrid(0); - - igniteEx.cluster().active(true); - - long totalStartTimeInParallel = System.currentTimeMillis() - startTime; - - for (int i = 0; i < CACHES_COUNT; i++) - igniteEx.cache(STATIC_CACHE_PREFIX + i).put(i, i); - - stopAllGrids(); - - assertTrue("Consistently cache stat time : " + totalStartTimeConsistently + - "Parallelization cache stat time : " + totalStartTimeInParallel, - totalStartTimeConsistently > totalStartTimeInParallel); - } - - /** - * Checking that start dynamic caches in parallel faster than consistenly. - * - * @throws Exception if fail. - */ - public void testParallelizationAcceleratesStartOfCaches2() throws Exception { - //prepare dynamic caches. - isStaticCache = false; - - IgniteEx igniteEx = startGrid(0); - - igniteEx.cluster().active(true); - - for (int i = 0; i < CACHES_COUNT; i++) - igniteEx.getOrCreateCache(DYNAMIC_CACHE_PREFIX + i); - - stopAllGrids(); - - //start caches consistently. - System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "false"); - - igniteEx = startGrid(0); - long startTime = System.currentTimeMillis(); - - igniteEx.cluster().active(true); - - long totalStartTimeConsistently = System.currentTimeMillis() - startTime; - - for (int i = 0; i < CACHES_COUNT; i++) - igniteEx.cache(DYNAMIC_CACHE_PREFIX + i); - - stopAllGrids(); - - //start caches in parallel. - System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "true"); - - startTime = System.currentTimeMillis(); - - igniteEx = startGrid(0); - - igniteEx.cluster().active(true); - - long totalStartTimeInParallel = System.currentTimeMillis() - startTime; - - for (int i = 0; i < CACHES_COUNT; i++) - igniteEx.cache(DYNAMIC_CACHE_PREFIX + i).put(i, i); - - stopAllGrids(); - - assertTrue("Consistently cache stat time : " + totalStartTimeConsistently + - "Parallelization cache stat time : " + totalStartTimeInParallel, - totalStartTimeConsistently > totalStartTimeInParallel); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java index 58941bc2f4ea3..6c48ecc2da6f7 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java @@ -35,7 +35,6 @@ import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest; import org.apache.ignite.internal.processors.cache.distributed.CachePageWriteLockUnlockTest; import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest; -import org.apache.ignite.internal.processors.cache.distributed.CacheStartInParallelTest; import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest; import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest; @@ -100,8 +99,6 @@ public static TestSuite suite(Set ignoredTests) throws Exception { suite.addTestSuite(CacheRentingStateRepairTest.class); - suite.addTestSuite(CacheStartInParallelTest.class); - suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class); suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class); From 94f094f61eae8438c007f7b4d95bb1e7fd4af3ae Mon Sep 17 00:00:00 2001 From: vd-pyatkov Date: Mon, 22 Oct 2018 18:19:53 +0300 Subject: [PATCH 032/403] IGNITE-9738 Client node can suddenly fail on start - Fixes #4968. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit d82b21ec56a956fa7cc5374e3f15e279e7c492ac) --- .../ignite/spi/discovery/tcp/ClientImpl.java | 10 +- .../LongClientConnectToClusterTest.java | 173 ++++++++++++++++++ .../IgniteSpiDiscoverySelfTestSuite.java | 2 + 3 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 3b57457b8e644..23e2f4c251b8e 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -305,6 +305,11 @@ class ClientImpl extends TcpDiscoveryImpl { } }.start(); + timer.schedule( + new MetricsSender(), + spi.metricsUpdateFreq, + spi.metricsUpdateFreq); + try { joinLatch.await(); @@ -317,11 +322,6 @@ class ClientImpl extends TcpDiscoveryImpl { throw new IgniteSpiException("Thread has been interrupted.", e); } - timer.schedule( - new MetricsSender(), - spi.metricsUpdateFreq, - spi.metricsUpdateFreq); - spi.printStartInfo(); } diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java new file mode 100644 index 0000000000000..a079926cff7ca --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery; + +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.EventType; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; +import java.io.IOException; +import java.io.OutputStream; +import java.net.Socket; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +/** + * Test client connects to two nodes cluster during time more than the + * {@link org.apache.ignite.configuration.IgniteConfiguration#clientFailureDetectionTimeout}. + */ +public class LongClientConnectToClusterTest extends GridCommonAbstractTest { + /** Client instance name. */ + public static final String CLIENT_INSTANCE_NAME = "client"; + /** Client metrics update count. */ + private static volatile int clientMetricsUpdateCnt; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + TcpDiscoverySpi discoSpi = getTestIgniteInstanceName(0).equals(igniteInstanceName) + ? new DelayedTcpDiscoverySpi() + : getTestIgniteInstanceName(1).equals(igniteInstanceName) + ? new UpdateMetricsInterceptorTcpDiscoverySpi() + : new TcpDiscoverySpi(); + + return super.getConfiguration(igniteInstanceName) + .setClientMode(igniteInstanceName.startsWith(CLIENT_INSTANCE_NAME)) + .setClientFailureDetectionTimeout(1_000) + .setMetricsUpdateFrequency(500) + .setDiscoverySpi(discoSpi + .setReconnectCount(1) + .setLocalAddress("127.0.0.1") + .setIpFinder(new TcpDiscoveryVmIpFinder() + .setAddresses(Collections.singletonList(igniteInstanceName.startsWith(CLIENT_INSTANCE_NAME) + ? "127.0.0.1:47501" + : "127.0.0.1:47500..47502")))); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + startGrids(2); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * Test method. + * + * @throws Exception If failed. + */ + public void testClientConnectToCluster() throws Exception { + clientMetricsUpdateCnt = 0; + + IgniteEx client = startGrid(CLIENT_INSTANCE_NAME); + + assertTrue(clientMetricsUpdateCnt > 0); + + assertTrue(client.localNode().isClient()); + + assertEquals(client.cluster().nodes().size(), 3); + } + + /** Discovery SPI which intercept TcpDiscoveryClientMetricsUpdateMessage. */ + private static class UpdateMetricsInterceptorTcpDiscoverySpi extends TcpDiscoverySpi { + /** */ + private class DiscoverySpiListenerWrapper implements DiscoverySpiListener { + /** */ + private DiscoverySpiListener delegate; + + /** + * @param delegate Delegate. + */ + private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) { + this.delegate = delegate; + } + + /** {@inheritDoc} */ + @Override public IgniteFuture onDiscovery( + int type, + long topVer, + ClusterNode node, + Collection topSnapshot, + @Nullable Map> topHist, + @Nullable DiscoverySpiCustomMessage spiCustomMsg + ) { + if (EventType.EVT_NODE_METRICS_UPDATED == type) { + log.info("Metrics update message catched from node " + node); + + assertFalse(locNode.isClient()); + + if (node.isClient()) + clientMetricsUpdateCnt++; + } + + if (delegate != null) + return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg); + + return new IgniteFinishedFutureImpl<>(); + } + + /** {@inheritDoc} */ + @Override public void onLocalNodeInitialized(ClusterNode locNode) { + if (delegate != null) + delegate.onLocalNodeInitialized(locNode); + } + } + + /** {@inheritDoc} */ + @Override public void setListener(@Nullable DiscoverySpiListener lsnr) { + super.setListener(new DiscoverySpiListenerWrapper(lsnr)); + } + } + + /** Discovery SPI delayed TcpDiscoveryNodeAddFinishedMessage. */ + private static class DelayedTcpDiscoverySpi extends TcpDiscoverySpi { + /** Delay message period millis. */ + public static final int DELAY_MSG_PERIOD_MILLIS = 2_000; + + /** {@inheritDoc} */ + @Override protected void writeToSocket(ClusterNode node, Socket sock, OutputStream out, + TcpDiscoveryAbstractMessage msg, long timeout) throws IOException, IgniteCheckedException { + if (msg instanceof TcpDiscoveryNodeAddFinishedMessage && msg.topologyVersion() == 3) { + log.info("Catched discovery message: " + msg); + + try { + Thread.sleep(DELAY_MSG_PERIOD_MILLIS); + } + catch (InterruptedException e) { + log.error("Interrupt on DelayedTcpDiscoverySpi.", e); + + Thread.currentThread().interrupt(); + } + } + + super.writeToSocket(node, sock, out, msg, timeout); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java index 04869f9aa91d8..80f093dfd68c2 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java @@ -23,6 +23,7 @@ import org.apache.ignite.spi.discovery.AuthenticationRestartTest; import org.apache.ignite.spi.discovery.FilterDataForClientNodeDiscoveryTest; import org.apache.ignite.spi.discovery.IgniteDiscoveryCacheReuseSelfTest; +import org.apache.ignite.spi.discovery.LongClientConnectToClusterTest; import org.apache.ignite.spi.discovery.tcp.DiscoveryUnmarshalVulnerabilityTest; import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest; import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest; @@ -97,6 +98,7 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(GridTcpSpiForwardingSelfTest.class)); suite.addTest(new TestSuite(TcpClientDiscoverySpiSelfTest.class)); + suite.addTest(new TestSuite(LongClientConnectToClusterTest.class)); suite.addTest(new TestSuite(TcpClientDiscoveryMarshallerCheckSelfTest.class)); suite.addTest(new TestSuite(TcpClientDiscoverySpiMulticastTest.class)); suite.addTest(new TestSuite(TcpClientDiscoverySpiFailureTimeoutSelfTest.class)); From 42ea30da4385e321c5fe904925eb3e3ac3c555cc Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Tue, 23 Oct 2018 17:57:04 +0300 Subject: [PATCH 033/403] Revert "IGNITE-5795 Register binary metadata during cache start - Fixes #4852." This reverts commit e52752a2319cfadf8ef794dd907bae5915ce6137. --- .../apache/ignite/internal/IgniteKernal.java | 2 +- .../binary/BinaryCachingMetadataHandler.java | 25 +- .../ignite/internal/binary/BinaryContext.java | 59 +--- .../binary/BinaryMetadataHandler.java | 10 - .../binary/BinaryNoopMetadataHandler.java | 6 - .../builder/BinaryObjectBuilderImpl.java | 2 +- .../internal/client/thin/TcpIgniteClient.java | 6 - .../processors/cache/GridCacheProcessor.java | 9 +- .../CacheObjectBinaryProcessorImpl.java | 5 - .../processors/query/GridQueryProcessor.java | 77 +---- .../binary/TestCachingMetadataHandler.java | 6 - .../CacheRegisterMetadataLocallyTest.java | 287 ------------------ .../cache/index/AbstractSchemaSelfTest.java | 6 +- .../index/H2DynamicIndexAbstractSelfTest.java | 48 ++- .../IgniteCacheWithIndexingTestSuite.java | 2 - 15 files changed, 56 insertions(+), 494 deletions(-) delete mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 40347d71473e6..c6ec9be1e1848 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1005,7 +1005,6 @@ public void start( // Start processors before discovery manager, so they will // be able to start receiving messages once discovery completes. try { - startProcessor(new GridMarshallerMappingProcessor(ctx)); startProcessor(new PdsConsistentIdProcessor(ctx)); startProcessor(new MvccProcessorImpl(ctx)); startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx)); @@ -1029,6 +1028,7 @@ public void start( startProcessor(createHadoopComponent()); startProcessor(new DataStructuresProcessor(ctx)); startProcessor(createComponent(PlatformProcessor.class, ctx)); + startProcessor(new GridMarshallerMappingProcessor(ctx)); // Start plugins. for (PluginProvider provider : ctx.plugins().allProviders()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java index b60dc097aaa1e..a0559cbdc8ba1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java @@ -46,28 +46,23 @@ private BinaryCachingMetadataHandler() { } /** {@inheritDoc} */ - @Override public synchronized void addMeta(int typeId, BinaryType type, - boolean failIfUnregistered) throws BinaryObjectException { - BinaryType oldType = metas.put(typeId, type); + @Override public synchronized void addMeta(int typeId, BinaryType type, boolean failIfUnregistered) throws BinaryObjectException { + synchronized (this) { + BinaryType oldType = metas.put(typeId, type); - if (oldType != null) { - BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata(); - BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata(); + if (oldType != null) { + BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata(); + BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata(); - BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta); + BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta); - BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context()); + BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context()); - metas.put(typeId, mergedType); + metas.put(typeId, mergedType); + } } } - /** {@inheritDoc} */ - @Override public synchronized void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) - throws BinaryObjectException { - addMeta(typeId, meta, failIfUnregistered); - } - /** {@inheritDoc} */ @Override public synchronized BinaryType metadata(int typeId) throws BinaryObjectException { return metas.get(typeId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java index 7ab74e09b82d0..7885d9575f32b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java @@ -617,18 +617,6 @@ else if (cpElement.isFile()) { */ public BinaryClassDescriptor descriptorForClass(Class cls, boolean deserialize, boolean failIfUnregistered) throws BinaryObjectException { - return descriptorForClass(cls, deserialize, failIfUnregistered, false); - } - - /** - * @param cls Class. - * @param failIfUnregistered Throw exception if class isn't registered. - * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all. - * @return Class descriptor. - * @throws BinaryObjectException In case of error. - */ - public BinaryClassDescriptor descriptorForClass(Class cls, boolean deserialize, boolean failIfUnregistered, - boolean onlyLocReg) throws BinaryObjectException { assert cls != null; BinaryClassDescriptor desc = descByCls.get(cls); @@ -637,7 +625,7 @@ public BinaryClassDescriptor descriptorForClass(Class cls, boolean deserializ if (failIfUnregistered) throw new UnregisteredClassException(cls); - desc = registerClassDescriptor(cls, deserialize, onlyLocReg); + desc = registerClassDescriptor(cls, deserialize); } else if (!desc.registered()) { if (!desc.userType()) { @@ -674,7 +662,7 @@ else if (!desc.registered()) { if (failIfUnregistered) throw new UnregisteredClassException(cls); - desc = registerUserClassDescriptor(desc, onlyLocReg); + desc = registerUserClassDescriptor(desc); } } @@ -727,7 +715,7 @@ public BinaryClassDescriptor descriptorForTypeId( } if (desc == null) { - desc = registerClassDescriptor(cls, deserialize, false); + desc = registerClassDescriptor(cls, deserialize); assert desc.typeId() == typeId : "Duplicate typeId [typeId=" + typeId + ", cls=" + cls + ", desc=" + desc + "]"; @@ -740,10 +728,9 @@ public BinaryClassDescriptor descriptorForTypeId( * Creates and registers {@link BinaryClassDescriptor} for the given {@code class}. * * @param cls Class. - * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all. * @return Class descriptor. */ - private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean deserialize, boolean onlyLocReg) { + private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean deserialize) { BinaryClassDescriptor desc; String clsName = cls.getName(); @@ -772,7 +759,7 @@ private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean dese desc = old; } else - desc = registerUserClassDescriptor(cls, deserialize, onlyLocReg); + desc = registerUserClassDescriptor(cls, deserialize); return desc; } @@ -781,10 +768,9 @@ private BinaryClassDescriptor registerClassDescriptor(Class cls, boolean dese * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}. * * @param cls Class. - * @param onlyLocReg {@code true} if descriptor need to register only locally. * @return Class descriptor. */ - private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean deserialize, boolean onlyLocReg) { + private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean deserialize) { boolean registered; final String clsName = cls.getName(); @@ -795,7 +781,7 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean final int typeId = mapper.typeId(clsName); - registered = registerUserClassName(typeId, cls.getName(), onlyLocReg); + registered = registerUserClassName(typeId, cls.getName()); BinarySerializer serializer = serializerForClass(cls); @@ -813,22 +799,9 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean registered ); - if (!deserialize) { - BinaryMetadata binaryMetadata = new BinaryMetadata( - typeId, - typeName, - desc.fieldsMeta(), - affFieldName, - null, - desc.isEnum(), - cls.isEnum() ? enumMap(cls) : null - ); - - if (onlyLocReg) - metaHnd.addMetaLocally(typeId, binaryMetadata.wrap(this), false); - else - metaHnd.addMeta(typeId, binaryMetadata.wrap(this), false); - } + if (!deserialize) + metaHnd.addMeta(typeId, new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), affFieldName, null, + desc.isEnum(), cls.isEnum() ? enumMap(cls) : null).wrap(this), false); descByCls.put(cls, desc); @@ -841,13 +814,12 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class cls, boolean * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}. * * @param desc Old descriptor that should be re-registered. - * @param onlyLocReg {@code true} if descriptor need to register only locally. * @return Class descriptor. */ - private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc, boolean onlyLocReg) { + private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc) { boolean registered; - registered = registerUserClassName(desc.typeId(), desc.describedClass().getName(), onlyLocReg); + registered = registerUserClassName(desc.typeId(), desc.describedClass().getName()); if (registered) { BinarySerializer serializer = desc.initialSerializer(); @@ -1219,18 +1191,15 @@ public void registerUserTypesSchema() { * * @param typeId Type ID. * @param clsName Class Name. - * @param onlyLocReg {@code true} if descriptor need to register only locally. * @return {@code True} if the mapping was registered successfully. */ - public boolean registerUserClassName(int typeId, String clsName, boolean onlyLocReg) { + public boolean registerUserClassName(int typeId, String clsName) { IgniteCheckedException e = null; boolean res = false; try { - res = onlyLocReg - ? marshCtx.registerClassNameLocally(JAVA_ID, typeId, clsName) - : marshCtx.registerClassName(JAVA_ID, typeId, clsName); + res = marshCtx.registerClassName(JAVA_ID, typeId, clsName); } catch (DuplicateTypeIdException dupEx) { // Ignore if trying to register mapped type name of the already registered class name and vise versa diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java index d1336bf6ae4a0..85ab1372f49f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java @@ -35,16 +35,6 @@ public interface BinaryMetadataHandler { */ public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException; - /** - * Adds meta data locally on current node without sending any messages. - * - * @param typeId Type ID. - * @param meta Metadata. - * @param failIfUnregistered Fail if unregistered. - * @throws BinaryObjectException In case of error. - */ - public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException; - /** * Gets meta data for provided type ID. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java index a552d611ea990..4ee24285c7ee5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java @@ -47,12 +47,6 @@ private BinaryNoopMetadataHandler() { // No-op. } - /** {@inheritDoc} */ - @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) - throws BinaryObjectException { - // No-op. - } - /** {@inheritDoc} */ @Override public BinaryType metadata(int typeId) throws BinaryObjectException { return null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index 5414a25f3fa63..abd63cd7b53ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -364,7 +364,7 @@ else if (readCache == null) { if (affFieldName0 == null) affFieldName0 = ctx.affinityKeyFieldName(typeId); - ctx.registerUserClassName(typeId, typeName, false); + ctx.registerUserClassName(typeId, typeName); ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldsMeta, affFieldName0, Collections.singleton(curSchema), false, null), writer.failIfUnregistered()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java index 856f41c4a324b..50408168d3118 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java @@ -262,12 +262,6 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler { cache.addMeta(typeId, meta, failIfUnregistered); // merge } - /** {@inheritDoc} */ - @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) - throws BinaryObjectException { - throw new UnsupportedOperationException("Can't register metadata locally for thin client."); - } - /** {@inheritDoc} */ @Override public BinaryType metadata(int typeId) throws BinaryObjectException { BinaryType meta = cache.metadata(typeId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 59703c06a6412..4f8301e405f58 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -4066,13 +4066,8 @@ else if (msg0 instanceof WalStateFinishMessage) return msg0.needExchange(); } - if (msg instanceof DynamicCacheChangeBatch) { - boolean changeRequested = cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer); - - ctx.query().onCacheChangeRequested((DynamicCacheChangeBatch)msg); - - return changeRequested; - } + if (msg instanceof DynamicCacheChangeBatch) + return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer); if (msg instanceof DynamicCacheChangeFailureMessage) cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg, topVer); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index f3078cb18df80..137db9f887dc5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -207,11 +207,6 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) { CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx), failIfUnregistered); } - @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) - throws BinaryObjectException { - CacheObjectBinaryProcessorImpl.this.addMetaLocally(typeId, meta); - } - @Override public BinaryType metadata(int typeId) throws BinaryObjectException { return CacheObjectBinaryProcessorImpl.this.metadata(typeId); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 366c62a240c9c..b5b104d2c2f51 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -17,8 +17,6 @@ package org.apache.ignite.internal.processors.query; -import javax.cache.Cache; -import javax.cache.CacheException; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; @@ -36,6 +34,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; +import javax.cache.Cache; +import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; @@ -62,20 +62,16 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; -import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch; -import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.KeyCacheObject; -import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture; import org.apache.ignite.internal.processors.cache.query.CacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; -import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.query.property.QueryBinaryProperty; import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheFilter; import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor; @@ -261,8 +257,6 @@ public GridQueryProcessor(GridKernalContext ctx) throws IgniteCheckedException { ctxs.queries().evictDetailMetrics(); } }, QRY_DETAIL_METRICS_EVICTION_FREQ, QRY_DETAIL_METRICS_EVICTION_FREQ); - - registerMetadataForRegisteredCaches(); } /** {@inheritDoc} */ @@ -909,73 +903,6 @@ public void skipFieldLookup(boolean skipFieldLookup) { this.skipFieldLookup = skipFieldLookup; } - /** - * Register metadata locally for already registered caches. - */ - private void registerMetadataForRegisteredCaches() { - for (DynamicCacheDescriptor cacheDescriptor : ctx.cache().cacheDescriptors().values()) { - registerBinaryMetadata(cacheDescriptor.cacheConfiguration(), cacheDescriptor.schema()); - } - } - - /** - * Handle of cache change request. - * - * @param batch Dynamic cache change batch request. - */ - public void onCacheChangeRequested(DynamicCacheChangeBatch batch) { - for (DynamicCacheChangeRequest req : batch.requests()) { - if (!req.start()) - continue; - - registerBinaryMetadata(req.startCacheConfiguration(), req.schema()); - } - } - - /** - * Register binary metadata locally. - * - * @param ccfg Cache configuration. - * @param schema Schema for which register metadata is required. - */ - private void registerBinaryMetadata(CacheConfiguration ccfg, QuerySchema schema) { - if (schema != null) { - Collection qryEntities = schema.entities(); - - if (!F.isEmpty(qryEntities)) { - boolean binaryEnabled = ctx.cacheObjects().isBinaryEnabled(ccfg); - - if (binaryEnabled) { - for (QueryEntity qryEntity : qryEntities) { - Class keyCls = U.box(U.classForName(qryEntity.findKeyType(), null, true)); - Class valCls = U.box(U.classForName(qryEntity.findValueType(), null, true)); - - if (keyCls != null) - registerDescriptorLocallyIfNeeded(keyCls); - - if (valCls != null) - registerDescriptorLocallyIfNeeded(valCls); - } - } - } - } - } - - /** - * Register class metadata locally if it didn't do it earlier. - * - * @param cls Class for which the metadata should be registered. - */ - private void registerDescriptorLocallyIfNeeded(Class cls) { - IgniteCacheObjectProcessor cacheObjProc = ctx.cacheObjects(); - - if (cacheObjProc instanceof CacheObjectBinaryProcessorImpl) { - ((CacheObjectBinaryProcessorImpl)cacheObjProc) - .binaryContext() - .descriptorForClass(cls, false, false, true); - } - } - /** * Handle custom discovery message. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java index 47138ddeb0806..c515f8191766a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java @@ -38,12 +38,6 @@ public class TestCachingMetadataHandler implements BinaryMetadataHandler { TestCachingMetadataHandler.class.getSimpleName() + '.'); } - /** {@inheritDoc} */ - @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) - throws BinaryObjectException { - addMeta(typeId, meta, failIfUnregistered); - } - /** {@inheritDoc} */ @Override public BinaryType metadata(int typeId) throws BinaryObjectException { return metas.get(typeId); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java deleted file mode 100644 index d4066c2f546f6..0000000000000 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import java.util.Collections; -import java.util.concurrent.ConcurrentLinkedQueue; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteException; -import org.apache.ignite.binary.BinaryType; -import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.affinity.AffinityKeyMapped; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.managers.communication.GridIoMessage; -import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper; -import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage; -import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage; -import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage; -import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.spi.IgniteSpiException; -import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; -import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -/** - * Tests, that binary metadata is registered correctly during the start without extra request to grid. - */ -public class CacheRegisterMetadataLocallyTest extends GridCommonAbstractTest { - /** */ - private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** */ - private static final String STATIC_CACHE_NAME = "staticCache"; - - /** */ - private static final String DYNAMIC_CACHE_NAME = "dynamicCache"; - - /** Holder of sent custom messages. */ - private final ConcurrentLinkedQueue customMessages = new ConcurrentLinkedQueue<>(); - - /** Holder of sent communication messages. */ - private final ConcurrentLinkedQueue communicationMessages = new ConcurrentLinkedQueue<>(); - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setDiscoverySpi(new TcpDiscoverySpi() { - @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException { - if (msg instanceof CustomMessageWrapper) - customMessages.add(((CustomMessageWrapper)msg).delegate()); - else - customMessages.add(msg); - - super.sendCustomEvent(msg); - } - }); - - cfg.setCommunicationSpi(new TcpCommunicationSpi() { - @Override public void sendMessage(ClusterNode node, Message msg, - IgniteInClosure ackC) throws IgniteSpiException { - if (msg instanceof GridIoMessage) - communicationMessages.add(((GridIoMessage)msg).message()); - - super.sendMessage(node, msg, ackC); - } - - @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { - if (msg instanceof GridIoMessage) - communicationMessages.add(((GridIoMessage)msg).message()); - - super.sendMessage(node, msg); - } - }); - - ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); - - if (igniteInstanceName.equals("client")) - cfg.setClientMode(true); - - cfg.setCacheConfiguration(cacheConfiguration(STATIC_CACHE_NAME, StaticKey.class, StaticValue.class)); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopAllGrids(); - - cleanPersistenceDir(); - - customMessages.clear(); - communicationMessages.clear(); - } - - /** - * @throws Exception If failed. - */ - public void testAffinityKeyRegisteredStaticCache() throws Exception { - Ignite ignite = startGrid(); - - assertEquals("affKey", getAffinityKey(ignite, StaticKey.class)); - assertEquals("affKey", getAffinityKey(ignite, StaticValue.class)); - } - - /** - * @throws Exception If failed. - */ - public void testAffinityKeyRegisteredDynamicCache() throws Exception { - Ignite ignite = startGrid(); - - ignite.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class)); - - assertEquals("affKey", getAffinityKey(ignite, DynamicKey.class)); - assertEquals("affKey", getAffinityKey(ignite, DynamicValue.class)); - } - - /** - * @throws Exception If failed. - */ - public void testClientFindsValueByAffinityKeyStaticCacheWithoutExtraRequest() throws Exception { - Ignite srv = startGrid(); - IgniteCache cache = srv.cache(STATIC_CACHE_NAME); - - testClientFindsValueByAffinityKey(cache, new StaticKey(1), new StaticValue(2)); - - assertCustomMessages(2); //MetadataUpdateProposedMessage for update schema. - assertCommunicationMessages(); - } - - /** - * @throws Exception If failed. - */ - public void testClientFindsValueByAffinityKeyDynamicCacheWithoutExtraRequest() throws Exception { - Ignite srv = startGrid(); - IgniteCache cache = - srv.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class)); - - testClientFindsValueByAffinityKey(cache, new DynamicKey(3), new DynamicValue(4)); - - //Expected only DynamicCacheChangeBatch for start cache and MetadataUpdateProposedMessage for update schema. - assertCustomMessages(3); - assertCommunicationMessages(); - } - - /** - * @param ignite Ignite instance. - * @param keyCls Key class. - * @return Name of affinity key field of the given class. - */ - private String getAffinityKey(Ignite ignite, Class keyCls) { - BinaryType binType = ignite.binary().type(keyCls); - - return binType.affinityKeyFieldName(); - } - - /** - * @param cache Cache instance. - * @param key Test key. - * @param val Test value. - * @throws Exception If failed. - */ - private void testClientFindsValueByAffinityKey(IgniteCache cache, K key, V val) throws Exception { - cache.put(key, val); - - assertTrue(cache.containsKey(key)); - - Ignite client = startGrid("client"); - - IgniteCache clientCache = client.cache(cache.getName()); - - assertTrue(clientCache.containsKey(key)); - } - - /** - * @param name Cache name. - * @param keyCls Key {@link Class}. - * @param valCls Value {@link Class}. - * @param Key type. - * @param Value type. - * @return Cache configuration - */ - private static CacheConfiguration cacheConfiguration(String name, Class keyCls, Class valCls) { - CacheConfiguration cfg = new CacheConfiguration<>(name); - cfg.setQueryEntities(Collections.singleton(new QueryEntity(keyCls, valCls))); - return cfg; - } - - /** - * Expecting that "proposed binary metadata"( {@link org.apache.ignite.internal.processors.marshaller.MappingProposedMessage}, - * {@link org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage}) will be skipped because - * it should be register locally during the start. - * - * @param expMsgCnt Count of expected messages. - */ - private void assertCustomMessages(int expMsgCnt) { - assertEquals(customMessages.toString(), expMsgCnt, customMessages.size()); - - customMessages.forEach(cm -> assertTrue(cm.toString(), cm instanceof DynamicCacheChangeBatch || cm instanceof MetadataUpdateProposedMessage)); - } - - /** - * Expecting that extra request to binary metadata( {@link MetadataRequestMessage}, {@link MetadataResponseMessage}) - * will be skipped because it should be register locally during the start. - */ - private void assertCommunicationMessages() { - communicationMessages.forEach(cm -> - assertFalse(cm.toString(), cm instanceof MetadataRequestMessage || cm instanceof MetadataResponseMessage) - ); - } - - /** */ - private static class StaticKey { - /** */ - @AffinityKeyMapped - private int affKey; - - /** - * @param affKey Affinity key. - */ - StaticKey(int affKey) { - this.affKey = affKey; - } - } - - /** */ - private static class StaticValue { - /** */ - @AffinityKeyMapped - private int affKey; - - /** - * @param affKey Affinity key. - */ - StaticValue(int affKey) { - } - } - - /** */ - private static class DynamicKey { - /** */ - @AffinityKeyMapped - private int affKey; - - /** - * @param affKey Affinity key. - */ - DynamicKey(int affKey) { - this.affKey = affKey; - } - } - - /** */ - private static class DynamicValue { - /** */ - @AffinityKeyMapped - private int affKey; - - /** - * @param affKey Affinity key. - */ - DynamicValue(int affKey) { - this.affKey = affKey; - } - } -} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java index 5f0e18e900617..7f1e2e74e8ec1 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java @@ -629,21 +629,21 @@ public long id() { public static class ValueClass { /** Field 1. */ @QuerySqlField - private Long field1; + private String field1; /** * Constructor. * * @param field1 Field 1. */ - public ValueClass(Long field1) { + public ValueClass(String field1) { this.field1 = field1; } /** * @return Field 1 */ - public Long field1() { + public String field1() { return field1; } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java index 07cb96b3c17de..10ef56fcd3b5e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java @@ -60,9 +60,9 @@ public abstract class H2DynamicIndexAbstractSelfTest extends AbstractSchemaSelfT IgniteCache cache = client().cache(CACHE_NAME); - cache.put(new KeyClass(1), new ValueClass(1L)); - cache.put(new KeyClass(2), new ValueClass(2L)); - cache.put(new KeyClass(3), new ValueClass(3L)); + cache.put(new KeyClass(1), new ValueClass("val1")); + cache.put(new KeyClass(2), new ValueClass("val2")); + cache.put(new KeyClass(3), new ValueClass("val3")); } /** {@inheritDoc} */ @@ -89,14 +89,14 @@ public void testCreateIndex() throws Exception { continue; List> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " + - "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll(); + "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll(); assertEquals(F.asList( Collections.singletonList("SELECT\n" + " \"id\"\n" + "FROM \"cache\".\"ValueClass\"\n" + - " /* \"cache\".\"idx_1\": \"field1\" = 1 */\n" + - "WHERE \"field1\" = 1") + " /* \"cache\".\"idx_1\": \"field1\" = 'A' */\n" + + "WHERE \"field1\" = 'A'") ), locRes); } @@ -106,7 +106,7 @@ public void testCreateIndex() throws Exception { assertSize(2); - cache.put(new KeyClass(4), new ValueClass(1L)); + cache.put(new KeyClass(4), new ValueClass("someVal")); assertSize(3); } @@ -162,14 +162,14 @@ public void testDropIndex() { continue; List> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " + - "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll(); + "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll(); assertEquals(F.asList( Collections.singletonList("SELECT\n" + " \"id\"\n" + "FROM \"cache\".\"ValueClass\"\n" + " /* \"cache\".\"ValueClass\".__SCAN_ */\n" + - "WHERE \"field1\" = 1") + "WHERE \"field1\" = 'A'") ), locRes); } @@ -204,39 +204,38 @@ public void testDropMissingIndexIfExists() { public void testIndexState() { IgniteCache cache = cache(); - assertColumnValues(1L, 2L, 3L); + assertColumnValues("val1", "val2", "val3"); cache.query(new SqlFieldsQuery("CREATE INDEX \"" + IDX_NAME_1_ESCAPED + "\" ON \"" + TBL_NAME_ESCAPED + "\"(\"" + FIELD_NAME_1_ESCAPED + "\" ASC)")); - assertColumnValues(1L, 2L, 3L); + assertColumnValues("val1", "val2", "val3"); cache.remove(new KeyClass(2)); - assertColumnValues(1L, 3L); + assertColumnValues("val1", "val3"); - cache.put(new KeyClass(0), new ValueClass(0L)); + cache.put(new KeyClass(0), new ValueClass("someVal")); - assertColumnValues(0L, 1L, 3L); + assertColumnValues("someVal", "val1", "val3"); cache.query(new SqlFieldsQuery("DROP INDEX \"" + IDX_NAME_1_ESCAPED + "\"")); - assertColumnValues(0L, 1L, 3L); + assertColumnValues("someVal", "val1", "val3"); } /** * Check that values of {@code field1} match what we expect. * @param vals Expected values. */ - private void assertColumnValues(Long... vals) { + private void assertColumnValues(String... vals) { List> expRes = new ArrayList<>(vals.length); - for (Long v : vals) + for (String v : vals) expRes.add(Collections.singletonList(v)); - List> all = cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" + - TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll(); - assertEquals(expRes, all); + assertEquals(expRes, cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" + + TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll()); } /** @@ -246,9 +245,8 @@ private void assertColumnValues(Long... vals) { private void assertSize(long expSize) { assertEquals(expSize, cache().size()); - Object actual = cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\"")) - .getAll().get(0).get(0); - assertEquals(expSize, actual); + assertEquals(expSize, cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\"")) + .getAll().get(0).get(0)); } /** @@ -315,8 +313,8 @@ private CacheConfiguration cacheConfiguration() { entity.setValueType(ValueClass.class.getName()); entity.addQueryField("id", Long.class.getName(), null); - entity.addQueryField(FIELD_NAME_1_ESCAPED, Long.class.getName(), null); - entity.addQueryField(FIELD_NAME_2_ESCAPED, Long.class.getName(), null); + entity.addQueryField(FIELD_NAME_1_ESCAPED, String.class.getName(), null); + entity.addQueryField(FIELD_NAME_2_ESCAPED, String.class.getName(), null); entity.setKeyFields(Collections.singleton("id")); diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index fae196f1df4c8..8517ebbf2a503 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -19,7 +19,6 @@ import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.BinaryTypeMismatchLoggingTest; -import org.apache.ignite.internal.processors.cache.CacheRegisterMetadataLocallyTest; import org.apache.ignite.internal.processors.cache.CacheBinaryKeyConcurrentQueryTest; import org.apache.ignite.internal.processors.cache.CacheConfigurationP2PTest; import org.apache.ignite.internal.processors.cache.CacheIndexStreamerTest; @@ -81,7 +80,6 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(CacheOperationsWithExpirationTest.class); suite.addTestSuite(CacheBinaryKeyConcurrentQueryTest.class); suite.addTestSuite(CacheQueryFilterExpiredTest.class); - suite.addTestSuite(CacheRegisterMetadataLocallyTest.class); suite.addTestSuite(ClientReconnectAfterClusterRestartTest.class); From 5b58e7680c2023ac20a9eab0f98fe2b41684984c Mon Sep 17 00:00:00 2001 From: Aleksei Scherbakov Date: Thu, 18 Oct 2018 14:52:34 +0300 Subject: [PATCH 034/403] IGNITE-9082 Throwing checked exception during tx commit without node stopping leads to data corruption - Fixes #4809. Signed-off-by: Ivan Rakov (cherry picked from commit 5eb871e191a14fc21f6e2c62bdfa742e27c14695) --- .../processors/cache/GridCacheMapEntry.java | 5 - .../cache/GridCacheSharedContext.java | 9 + .../GridDistributedTxRemoteAdapter.java | 546 +++++++++--------- .../dht/GridDhtTxFinishFuture.java | 11 +- .../cache/distributed/dht/GridDhtTxLocal.java | 10 +- .../dht/GridDhtTxLocalAdapter.java | 3 + .../dht/GridDhtTxPrepareFuture.java | 59 +- .../near/GridNearTxFinishFuture.java | 38 +- .../distributed/near/GridNearTxLocal.java | 2 +- .../cache/transactions/IgniteTxAdapter.java | 31 + .../cache/transactions/IgniteTxHandler.java | 119 ++-- .../transactions/IgniteTxLocalAdapter.java | 519 ++++++++--------- .../processors/failure/FailureProcessor.java | 8 + .../apache/ignite/spi/IgniteSpiAdapter.java | 1 - .../cache/GridCacheAbstractSelfTest.java | 9 +- .../cache/query/IndexingSpiQuerySelfTest.java | 66 +-- .../query/IndexingSpiQueryTxSelfTest.java | 74 ++- .../AbstractTransactionIntergrityTest.java | 111 ++-- ...tegrityWithPrimaryIndexCorruptionTest.java | 268 +++++---- ...ionIntegrityWithSystemWorkerDeathTest.java | 6 +- .../TxDataConsistencyOnCommitFailureTest.java | 234 ++++++++ .../junits/common/GridCommonAbstractTest.java | 19 +- .../testsuites/IgniteCacheTestSuite9.java | 3 + 23 files changed, 1226 insertions(+), 925 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 87de4b9d358c1..c753296b528da 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -2903,11 +2903,6 @@ protected final boolean markObsolete0(GridCacheVersion ver, boolean clear, GridC ver = newVer; flags &= ~IS_EVICT_DISABLED; - if (cctx.mvccEnabled()) - cctx.offheap().mvccRemoveAll(this); - else - removeValue(); - onInvalidate(); return obsoleteVersionExtras() != null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index 1f2f63691d9e6..efbbf072db3fd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -1141,4 +1141,13 @@ public boolean readOnlyMode() { public void readOnlyMode(boolean readOnlyMode) { this.readOnlyMode = readOnlyMode; } + + /** + * For test purposes. + * @param txMgr Tx manager. + */ + public void setTxManager(IgniteTxManager txMgr) { + this.txMgr = txMgr; + } + } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java index e095c08087067..51e6e66e76f6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java @@ -52,7 +52,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry; -import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; @@ -509,333 +508,302 @@ private void commitIfLocked() throws IgniteCheckedException { batchStoreCommit(writeMap().values()); - try { - // Node that for near transactions we grab all entries. - for (IgniteTxEntry txEntry : entries) { - GridCacheContext cacheCtx = txEntry.context(); + // Node that for near transactions we grab all entries. + for (IgniteTxEntry txEntry : entries) { + GridCacheContext cacheCtx = txEntry.context(); - boolean replicate = cacheCtx.isDrEnabled(); + boolean replicate = cacheCtx.isDrEnabled(); + while (true) { try { - while (true) { - try { - GridCacheEntryEx cached = txEntry.cached(); + GridCacheEntryEx cached = txEntry.cached(); - if (cached == null) - txEntry.cached(cached = cacheCtx.cache().entryEx(txEntry.key(), topologyVersion())); + if (cached == null) + txEntry.cached(cached = cacheCtx.cache().entryEx(txEntry.key(), topologyVersion())); - if (near() && cacheCtx.dr().receiveEnabled()) { - cached.markObsolete(xidVer); + if (near() && cacheCtx.dr().receiveEnabled()) { + cached.markObsolete(xidVer); - break; - } + break; + } - GridNearCacheEntry nearCached = null; + GridNearCacheEntry nearCached = null; - if (updateNearCache(cacheCtx, txEntry.key(), topVer)) - nearCached = cacheCtx.dht().near().peekExx(txEntry.key()); + if (updateNearCache(cacheCtx, txEntry.key(), topVer)) + nearCached = cacheCtx.dht().near().peekExx(txEntry.key()); - if (!F.isEmpty(txEntry.entryProcessors())) - txEntry.cached().unswap(false); + if (!F.isEmpty(txEntry.entryProcessors())) + txEntry.cached().unswap(false); - IgniteBiTuple res = - applyTransformClosures(txEntry, false, ret); + IgniteBiTuple res = + applyTransformClosures(txEntry, false, ret); - GridCacheOperation op = res.get1(); - CacheObject val = res.get2(); + GridCacheOperation op = res.get1(); + CacheObject val = res.get2(); - GridCacheVersion explicitVer = txEntry.conflictVersion(); + GridCacheVersion explicitVer = txEntry.conflictVersion(); - if (explicitVer == null) - explicitVer = writeVersion(); + if (explicitVer == null) + explicitVer = writeVersion(); - if (txEntry.ttl() == CU.TTL_ZERO) - op = DELETE; + if (txEntry.ttl() == CU.TTL_ZERO) + op = DELETE; - boolean conflictNeedResolve = cacheCtx.conflictNeedResolve(); + boolean conflictNeedResolve = cacheCtx.conflictNeedResolve(); - GridCacheVersionConflictContext conflictCtx = null; + GridCacheVersionConflictContext conflictCtx = null; - if (conflictNeedResolve) { - IgniteBiTuple - drRes = conflictResolve(op, txEntry, val, explicitVer, cached); + if (conflictNeedResolve) { + IgniteBiTuple + drRes = conflictResolve(op, txEntry, val, explicitVer, cached); - assert drRes != null; + assert drRes != null; - conflictCtx = drRes.get2(); + conflictCtx = drRes.get2(); - if (conflictCtx.isUseOld()) - op = NOOP; - else if (conflictCtx.isUseNew()) { - txEntry.ttl(conflictCtx.ttl()); - txEntry.conflictExpireTime(conflictCtx.expireTime()); - } - else if (conflictCtx.isMerge()) { - op = drRes.get1(); - val = txEntry.context().toCacheObject(conflictCtx.mergeValue()); - explicitVer = writeVersion(); + if (conflictCtx.isUseOld()) + op = NOOP; + else if (conflictCtx.isUseNew()) { + txEntry.ttl(conflictCtx.ttl()); + txEntry.conflictExpireTime(conflictCtx.expireTime()); + } + else if (conflictCtx.isMerge()) { + op = drRes.get1(); + val = txEntry.context().toCacheObject(conflictCtx.mergeValue()); + explicitVer = writeVersion(); - txEntry.ttl(conflictCtx.ttl()); - txEntry.conflictExpireTime(conflictCtx.expireTime()); - } - } - else - // Nullify explicit version so that innerSet/innerRemove will work as usual. - explicitVer = null; - - GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null; - - if (!near() && cacheCtx.group().persistenceEnabled() && cacheCtx.group().walEnabled() && - op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) { - if (dataEntries == null) - dataEntries = new ArrayList<>(entries.size()); - - dataEntries.add( - new T2<>( - new DataEntry( - cacheCtx.cacheId(), - txEntry.key(), - val, - op, - nearXidVersion(), - writeVersion(), - 0, - txEntry.key().partition(), - txEntry.updateCounter() - ), - txEntry - ) - ); - } + txEntry.ttl(conflictCtx.ttl()); + txEntry.conflictExpireTime(conflictCtx.expireTime()); + } + } + else + // Nullify explicit version so that innerSet/innerRemove will work as usual. + explicitVer = null; + + GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null; + + if (!near() && cacheCtx.group().persistenceEnabled() && cacheCtx.group().walEnabled() && + op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) { + if (dataEntries == null) + dataEntries = new ArrayList<>(entries.size()); + + dataEntries.add( + new T2<>( + new DataEntry( + cacheCtx.cacheId(), + txEntry.key(), + val, + op, + nearXidVersion(), + writeVersion(), + 0, + txEntry.key().partition(), + txEntry.updateCounter() + ), + txEntry + ) + ); + } - if (op == CREATE || op == UPDATE) { - // Invalidate only for near nodes (backups cannot be invalidated). - if (isSystemInvalidate() || (isInvalidate() && cacheCtx.isNear())) - cached.innerRemove(this, - eventNodeId(), - nodeId, - false, - true, - true, - txEntry.keepBinary(), - txEntry.hasOldValue(), - txEntry.oldValue(), - topVer, - null, - replicate ? DR_BACKUP : DR_NONE, - near() ? null : explicitVer, - CU.subjectId(this, cctx), - resolveTaskName(), - dhtVer, - txEntry.updateCounter(), - mvccSnapshot()); - else { - assert val != null : txEntry; - - GridCacheUpdateTxResult updRes = cached.innerSet(this, - eventNodeId(), - nodeId, - val, - false, - false, - txEntry.ttl(), - true, - true, - txEntry.keepBinary(), - txEntry.hasOldValue(), - txEntry.oldValue(), - topVer, - null, - replicate ? DR_BACKUP : DR_NONE, - txEntry.conflictExpireTime(), - near() ? null : explicitVer, - CU.subjectId(this, cctx), - resolveTaskName(), - dhtVer, - txEntry.updateCounter(), - mvccSnapshot()); - - txEntry.updateCounter(updRes.updateCounter()); - - if (updRes.loggedPointer() != null) - ptr = updRes.loggedPointer(); - - // Keep near entry up to date. - if (nearCached != null) { - CacheObject val0 = cached.valueBytes(); - - nearCached.updateOrEvict(xidVer, - val0, - cached.expireTime(), - cached.ttl(), - nodeId, - topVer); - } - } - } - else if (op == DELETE) { - GridCacheUpdateTxResult updRes = cached.innerRemove(this, - eventNodeId(), + if (op == CREATE || op == UPDATE) { + // Invalidate only for near nodes (backups cannot be invalidated). + if (isSystemInvalidate() || (isInvalidate() && cacheCtx.isNear())) + cached.innerRemove(this, + eventNodeId(), + nodeId, + false, + true, + true, + txEntry.keepBinary(), + txEntry.hasOldValue(), + txEntry.oldValue(), + topVer, + null, + replicate ? DR_BACKUP : DR_NONE, + near() ? null : explicitVer, + CU.subjectId(this, cctx), + resolveTaskName(), + dhtVer, + txEntry.updateCounter(), + mvccSnapshot()); + else { + assert val != null : txEntry; + + GridCacheUpdateTxResult updRes = cached.innerSet(this, + eventNodeId(), + nodeId, + val, + false, + false, + txEntry.ttl(), + true, + true, + txEntry.keepBinary(), + txEntry.hasOldValue(), + txEntry.oldValue(), + topVer, + null, + replicate ? DR_BACKUP : DR_NONE, + txEntry.conflictExpireTime(), + near() ? null : explicitVer, + CU.subjectId(this, cctx), + resolveTaskName(), + dhtVer, + txEntry.updateCounter(), + mvccSnapshot()); + + txEntry.updateCounter(updRes.updateCounter()); + + if (updRes.loggedPointer() != null) + ptr = updRes.loggedPointer(); + + // Keep near entry up to date. + if (nearCached != null) { + CacheObject val0 = cached.valueBytes(); + + nearCached.updateOrEvict(xidVer, + val0, + cached.expireTime(), + cached.ttl(), nodeId, - false, - true, - true, - txEntry.keepBinary(), - txEntry.hasOldValue(), - txEntry.oldValue(), - topVer, - null, - replicate ? DR_BACKUP : DR_NONE, - near() ? null : explicitVer, - CU.subjectId(this, cctx), - resolveTaskName(), - dhtVer, - txEntry.updateCounter(), - mvccSnapshot()); - - txEntry.updateCounter(updRes.updateCounter()); - - if (updRes.loggedPointer() != null) - ptr = updRes.loggedPointer(); - - // Keep near entry up to date. - if (nearCached != null) - nearCached.updateOrEvict(xidVer, null, 0, 0, nodeId, topVer); - } - else if (op == RELOAD) { - CacheObject reloaded = cached.innerReload(); - - if (nearCached != null) { - nearCached.innerReload(); - - nearCached.updateOrEvict(cached.version(), - reloaded, - cached.expireTime(), - cached.ttl(), - nodeId, - topVer); - } + topVer); } - else if (op == READ) { - assert near(); - - if (log.isDebugEnabled()) - log.debug("Ignoring READ entry when committing: " + txEntry); - } - // No-op. - else { - if (conflictCtx == null || !conflictCtx.isUseOld()) { - if (txEntry.ttl() != CU.TTL_NOT_CHANGED) - cached.updateTtl(null, txEntry.ttl()); - - if (nearCached != null) { - CacheObject val0 = cached.valueBytes(); - - nearCached.updateOrEvict(xidVer, - val0, - cached.expireTime(), - cached.ttl(), - nodeId, - topVer); - } - } - } - - // Assert after setting values as we want to make sure - // that if we replaced removed entries. - assert - txEntry.op() == READ || onePhaseCommit() || - // If candidate is not there, then lock was explicit - // and we simply allow the commit to proceed. - !cached.hasLockCandidateUnsafe(xidVer) || cached.lockedByUnsafe(xidVer) : - "Transaction does not own lock for commit [entry=" + cached + - ", tx=" + this + ']'; - - // Break out of while loop. - break; - } - catch (GridCacheEntryRemovedException ignored) { - if (log.isDebugEnabled()) - log.debug("Attempting to commit a removed entry (will retry): " + txEntry); - - // Renew cached entry. - txEntry.cached(cacheCtx.cache().entryEx(txEntry.key(), topologyVersion())); } } - } - catch (Throwable ex) { - boolean isNodeStopping = X.hasCause(ex, NodeStoppingException.class); - boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class); - - // In case of error, we still make the best effort to commit, - // as there is no way to rollback at this point. - err = new IgniteTxHeuristicCheckedException("Commit produced a runtime exception " + - "(all transaction entries will be invalidated): " + CU.txString(this), ex); - - if (isNodeStopping) { - U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this + - ", err=" + ex + ']'); - } - else if (hasInvalidEnvironmentIssue) { - U.warn(log, "Failed to commit transaction, node is in invalid state and will be stopped [tx=" + this + - ", err=" + ex + ']'); + else if (op == DELETE) { + GridCacheUpdateTxResult updRes = cached.innerRemove(this, + eventNodeId(), + nodeId, + false, + true, + true, + txEntry.keepBinary(), + txEntry.hasOldValue(), + txEntry.oldValue(), + topVer, + null, + replicate ? DR_BACKUP : DR_NONE, + near() ? null : explicitVer, + CU.subjectId(this, cctx), + resolveTaskName(), + dhtVer, + txEntry.updateCounter(), + mvccSnapshot()); + + txEntry.updateCounter(updRes.updateCounter()); + + if (updRes.loggedPointer() != null) + ptr = updRes.loggedPointer(); + + // Keep near entry up to date. + if (nearCached != null) + nearCached.updateOrEvict(xidVer, null, 0, 0, nodeId, topVer); } - else - U.error(log, "Commit failed.", err); - - state(UNKNOWN); - - if (hasInvalidEnvironmentIssue) - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex)); - else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or invalidation. - try { - // Courtesy to minimize damage. - uncommit(); + else if (op == RELOAD) { + CacheObject reloaded = cached.innerReload(); + + if (nearCached != null) { + nearCached.innerReload(); + + nearCached.updateOrEvict(cached.version(), + reloaded, + cached.expireTime(), + cached.ttl(), + nodeId, + topVer); } - catch (Throwable ex1) { - U.error(log, "Failed to uncommit transaction: " + this, ex1); + } + else if (op == READ) { + assert near(); - if (ex1 instanceof Error) - throw ex1; + if (log.isDebugEnabled()) + log.debug("Ignoring READ entry when committing: " + txEntry); + } + // No-op. + else { + if (conflictCtx == null || !conflictCtx.isUseOld()) { + if (txEntry.ttl() != CU.TTL_NOT_CHANGED) + cached.updateTtl(null, txEntry.ttl()); + + if (nearCached != null) { + CacheObject val0 = cached.valueBytes(); + + nearCached.updateOrEvict(xidVer, + val0, + cached.expireTime(), + cached.ttl(), + nodeId, + topVer); + } } } - if (ex instanceof Error) - throw (Error) ex; + // Assert after setting values as we want to make sure + // that if we replaced removed entries. + assert + txEntry.op() == READ || onePhaseCommit() || + // If candidate is not there, then lock was explicit + // and we simply allow the commit to proceed. + !cached.hasLockCandidateUnsafe(xidVer) || cached.lockedByUnsafe(xidVer) : + "Transaction does not own lock for commit [entry=" + cached + + ", tx=" + this + ']'; + + // Break out of while loop. + break; + } + catch (GridCacheEntryRemovedException ignored) { + if (log.isDebugEnabled()) + log.debug("Attempting to commit a removed entry (will retry): " + txEntry); - throw err; + // Renew cached entry. + txEntry.cached(cacheCtx.cache().entryEx(txEntry.key(), topologyVersion())); } } + } - // Apply cache size deltas. - applyTxSizes(); + // Apply cache size deltas. + applyTxSizes(); - TxCounters txCntrs = txCounters(false); + TxCounters txCntrs = txCounters(false); - // Apply update counters. - if (txCntrs != null) - cctx.tm().txHandler().applyPartitionsUpdatesCounters(txCntrs.updateCounters()); + // Apply update counters. + if (txCntrs != null) + cctx.tm().txHandler().applyPartitionsUpdatesCounters(txCntrs.updateCounters()); - cctx.mvccCaching().onTxFinished(this, true); + cctx.mvccCaching().onTxFinished(this, true); - if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null) { - // Set new update counters for data entries received from persisted tx entries. - List entriesWithCounters = dataEntries.stream() - .map(tuple -> tuple.get1().partitionCounter(tuple.get2().updateCounter())) - .collect(Collectors.toList()); + if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null) { + // Set new update counters for data entries received from persisted tx entries. + List entriesWithCounters = dataEntries.stream() + .map(tuple -> tuple.get1().partitionCounter(tuple.get2().updateCounter())) + .collect(Collectors.toList()); - cctx.wal().log(new DataRecord(entriesWithCounters)); - } + cctx.wal().log(new DataRecord(entriesWithCounters)); + } + + if (ptr != null && !cctx.tm().logTxRecords()) + cctx.wal().flush(ptr, false); + } + catch (Throwable ex) { + state(UNKNOWN); - if (ptr != null && !cctx.tm().logTxRecords()) - cctx.wal().flush(ptr, false); + if (X.hasCause(ex, NodeStoppingException.class)) { + U.warn(log, "Failed to commit transaction, node is stopping [tx=" + CU.txString(this) + + ", err=" + ex + ']'); + + return; } - catch (StorageException e) { - err = e; - throw new IgniteCheckedException("Failed to log transaction record " + - "(transaction will be rolled back): " + this, e); + err = heuristicException(ex); + + try { + uncommit(); + } + catch (Throwable e) { + err.addSuppressed(e); } + + throw err; } finally { cctx.database().checkpointReadUnlock(); @@ -875,9 +843,19 @@ else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or in throw new IgniteCheckedException("Invalid transaction state for commit [state=" + state + ", tx=" + this + ']'); rollbackRemoteTx(); + + return; } - commitIfLocked(); + try { + commitIfLocked(); + } + catch (IgniteTxHeuristicCheckedException e) { + // Treat heuristic exception as critical. + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + + throw e; + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java index 472aafd724d16..d0fbd90bf03eb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java @@ -173,10 +173,13 @@ public void rollbackOnError(Throwable e) { if (ERR_UPD.compareAndSet(this, null, e)) { tx.setRollbackOnly(); - if (X.hasCause(e, InvalidEnvironmentException.class, NodeStoppingException.class)) + if (X.hasCause(e, NodeStoppingException.class) || cctx.kernalContext().failure().nodeStopping()) onComplete(); - else + else { + // Rolling back a remote transaction may result in partial commit. + // This is only acceptable in tests with no-op failure handler. finish(false); + } } } @@ -230,9 +233,9 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) { if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) { try { - boolean hasInvalidEnvironmentIssue = X.hasCause(err, InvalidEnvironmentException.class, NodeStoppingException.class); + boolean nodeStopping = X.hasCause(err, NodeStoppingException.class); - this.tx.tmFinish(err == null, hasInvalidEnvironmentIssue, false); + this.tx.tmFinish(err == null, nodeStopping || cctx.kernalContext().failure().nodeStopping(), false); } catch (IgniteCheckedException finishErr) { U.error(log, "Failed to finish tx: " + tx, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java index a091d44ac423f..ca451f064eee3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java @@ -24,6 +24,8 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; @@ -39,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException; import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException; import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException; import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; @@ -46,6 +49,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; @@ -467,7 +471,11 @@ else if (!lockFut.isDone()) { ", tx=" + CU.txString(this) + ']'); } catch (IgniteCheckedException e) { - U.error(log, "Failed to finish transaction [commit=" + commit + ", tx=" + this + ']', e); + logTxFinishErrorSafe(log, commit, e); + + // Treat heuristic exception as critical. + if (X.hasCause(e, IgniteTxHeuristicCheckedException.class)) + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); err = e; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java index 29169162caa28..86f9c3c0d0cb8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java @@ -29,6 +29,8 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; @@ -41,6 +43,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; +import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java index 57280788bc16f..609bff801e406 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java @@ -36,6 +36,8 @@ import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.IgniteDiagnosticAware; import org.apache.ignite.internal.IgniteDiagnosticPrepareContext; import org.apache.ignite.internal.IgniteInternalFuture; @@ -740,8 +742,6 @@ private boolean mapIfLocked() { if (tx.commitOnPrepare()) { if (tx.markFinalizing(IgniteInternalTx.FinalizationStatus.USER_FINISH)) { - IgniteInternalFuture fut = null; - CIX1> resClo = new CIX1>() { @Override public void applyx(IgniteInternalFuture fut) { @@ -753,42 +753,43 @@ private boolean mapIfLocked() { } }; - if (prepErr == null) { - try { - fut = tx.commitAsync(); - } - catch (RuntimeException | Error e) { - Exception hEx = new IgniteTxHeuristicCheckedException("Commit produced a runtime " + - "exception: " + CU.txString(tx), e); - - res.error(hEx); + try { + if (prepErr == null) { + try { + tx.commitAsync().listen(resClo); + } + catch (Throwable e) { + res.error(e); - tx.systemInvalidate(true); + tx.systemInvalidate(true); - try { - fut = tx.rollbackAsync(); + try { + tx.rollbackAsync().listen(resClo); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } - fut.listen(resClo); + throw e; } - catch (Throwable e1) { - e.addSuppressed(e1); + } + else if (!cctx.kernalContext().isStopping()) { + try { + tx.rollbackAsync().listen(resClo); } + catch (Throwable e) { + if (err != null) + err.addSuppressed(e); - throw e; + throw err; + } } - } - else if (!cctx.kernalContext().isStopping()) - try { - fut = tx.rollbackAsync(); - } - catch (Throwable e) { - err.addSuppressed(e); - fut = null; - } + catch (Throwable e){ + tx.logTxFinishErrorSafe(log, true, e); - if (fut != null) - fut.listen(resClo); + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } } } else { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java index 4a4d8e3524932..befa3053fe605 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java @@ -311,7 +311,7 @@ void forceFinish() { if (err != null) { tx.setRollbackOnly(); - nodeStop = err instanceof NodeStoppingException; + nodeStop = err instanceof NodeStoppingException || cctx.kernalContext().failure().nodeStopping(); } if (commit) { @@ -357,29 +357,6 @@ else if (err != null) } if (super.onDone(tx0, err)) { - if (error() instanceof IgniteTxHeuristicCheckedException && !nodeStop) { - AffinityTopologyVersion topVer = tx.topologyVersion(); - - for (IgniteTxEntry e : tx.writeMap().values()) { - GridCacheContext cacheCtx = e.context(); - - try { - if (e.op() != NOOP && !cacheCtx.affinity().keyLocalNode(e.key(), topVer)) { - GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key()); - - if (entry != null) - entry.invalidate(tx.xidVersion()); - } - } - catch (Throwable t) { - U.error(log, "Failed to invalidate entry.", t); - - if (t instanceof Error) - throw (Error)t; - } - } - } - // Don't forget to clean up. cctx.mvcc().removeFuture(futId); @@ -402,8 +379,7 @@ private boolean isMini(IgniteInternalFuture fut) { } /** {@inheritDoc} */ - @Override @SuppressWarnings("ForLoopReplaceableByForEach") - public void finish(final boolean commit, final boolean clearThreadMap, final boolean onTimeout) { + @Override public void finish(final boolean commit, final boolean clearThreadMap, final boolean onTimeout) { if (!cctx.mvcc().addFuture(this, futureId())) return; @@ -490,18 +466,22 @@ private void doFinish(boolean commit, boolean clearThreadMap) { } } + // Cleanup transaction if heuristic failure. + if (tx.state() == UNKNOWN) + cctx.tm().rollbackTx(tx, clearThreadMap, false); + if ((tx.onePhaseCommit() && needFinishOnePhase(commit)) || (!tx.onePhaseCommit() && mappings != null)) { if (mappings.single()) { GridDistributedTxMapping mapping = mappings.singleMapping(); if (mapping != null) { - assert !hasFutures() || waitTxs != null : futures(); + assert !hasFutures() || isDone() || waitTxs != null : futures(); finish(1, mapping, commit, !clearThreadMap); } } else { - assert !hasFutures() || waitTxs != null : futures(); + assert !hasFutures() || isDone() || waitTxs != null : futures(); finish(mappings.mappings(), commit, !clearThreadMap); } @@ -762,7 +742,7 @@ private void readyNearMappingFromBackup(GridDistributedTxMapping mapping) { /** * @param mappings Mappings. * @param commit Commit flag. - * @param {@code true} If need to add completed version on finish. + * @param useCompletedVer {@code True} if need to add completed version on finish. */ private void finish(Iterable mappings, boolean commit, boolean useCompletedVer) { int miniId = 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java index 68aa5c05faf97..c7b2ad3a9e9b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java @@ -3955,7 +3955,7 @@ private IgniteInternalFuture chainFinishFuture(final NearTxFin assert rollbackFut.isDone() : rollbackFut; } - else + else // First finish attempt was unsuccessful. Try again. rollbackFut.finish(false, clearThreadMap, onTimeout); } else { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index ff2ade53b4b68..399359bf11ab7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -75,6 +75,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx; import org.apache.ignite.internal.processors.cluster.BaselineTopology; +import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException; import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException; import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; import org.apache.ignite.internal.util.GridSetWrapper; @@ -763,6 +764,36 @@ public final IgniteCheckedException rollbackException() { "[timeout=" + timeout() + ", tx=" + CU.txString(this) + ']'); } + /** + * @param ex Root cause. + */ + public final IgniteCheckedException heuristicException(Throwable ex) { + return new IgniteTxHeuristicCheckedException("Committing a transaction has produced runtime exception", ex); + } + + /** + * @param log Log. + * @param commit Commit. + * @param e Exception. + */ + public void logTxFinishErrorSafe(@Nullable IgniteLogger log, boolean commit, Throwable e) { + assert e != null : "Exception is expected"; + + final String fmt = "Failed completing the transaction: [commit=%s, tx=%s, plc=%s]"; + + try { + // First try printing a full transaction. This is error prone. + U.error(log, String.format(fmt, commit, this, + cctx.gridConfig().getFailureHandler().getClass().getSimpleName()), e); + } + catch (Throwable e0) { + e.addSuppressed(e0); + + U.error(log, String.format(fmt, commit, CU.txString(this), + cctx.gridConfig().getFailureHandler().getClass().getSimpleName()), e); + } + } + /** {@inheritDoc} */ @Override public GridCacheVersion xidVersion() { return xidVer; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index e4fee2fab38ff..75e208719689a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -1049,45 +1049,34 @@ private IgniteInternalFuture finishDhtLocal(UUID nodeId, } catch (Throwable e) { try { - U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e); - } - catch (Throwable e0) { - ClusterNode node0 = ctx.discovery().node(nodeId); - - U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + - CU.txString(tx) + ']', e); - - U.error(log, "Failed to log message due to an error: ", e0); + if (tx != null) { + tx.commitError(e); - if (node0 != null && (!node0.isClient() || node0.isLocal())) { - ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + tx.systemInvalidate(true); - throw e; - } - } - - if (tx != null) { - tx.commitError(e); - - tx.systemInvalidate(true); + try { + IgniteInternalFuture res = tx.rollbackDhtLocalAsync(); - try { - IgniteInternalFuture res = tx.rollbackDhtLocalAsync(); + // Only for error logging. + res.listen(CU.errorLogger(log)); - // Only for error logging. - res.listen(CU.errorLogger(log)); + return res; + } + catch (Throwable e1) { + e.addSuppressed(e1); + } - return res; + tx.logTxFinishErrorSafe(log, req.commit(), e); } - catch (Throwable e1) { - e.addSuppressed(e1); - } - } - if (e instanceof Error) - throw (Error)e; + if (e instanceof Error) + throw (Error)e; - return new GridFinishedFuture<>(e); + return new GridFinishedFuture<>(e); + } + finally { + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } } } @@ -1112,20 +1101,26 @@ public IgniteInternalFuture finishColocatedLocal(boolean commi return tx.rollbackAsyncLocal(); } catch (Throwable e) { - U.error(log, "Failed completing transaction [commit=" + commit + ", tx=" + tx + ']', e); - - if (e instanceof Error) - throw e; + try { + if (tx != null) { + try { + return tx.rollbackNearTxLocalAsync(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } - if (tx != null) - try { - return tx.rollbackNearTxLocalAsync(); - } - catch (Throwable e1) { - e.addSuppressed(e1); + tx.logTxFinishErrorSafe(log, commit, e); } - return new GridFinishedFuture<>(e); + if (e instanceof Error) + throw e; + + return new GridFinishedFuture<>(e); + } + finally { + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } } } @@ -1212,10 +1207,6 @@ else if (e instanceof IgniteTxOptimisticCheckedException) { if (log.isDebugEnabled()) log.debug("Optimistic failure for remote transaction (will rollback): " + req); } - else if (e instanceof IgniteTxHeuristicCheckedException) { - U.warn(log, "Failed to commit transaction (all transaction entries were invalidated): " + - CU.txString(dhtTx)); - } else U.error(log, "Failed to process prepare request: " + req, e); @@ -1440,9 +1431,10 @@ protected void finish( tx.rollbackRemoteTx(); } } + catch (IgniteTxHeuristicCheckedException e) { + // Already uncommitted. + } catch (Throwable e) { - U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e); - // Mark transaction for invalidate. tx.invalidate(true); tx.systemInvalidate(true); @@ -1460,6 +1452,8 @@ protected void finish( } /** + * Finish for one-phase distributed tx. + * * @param tx Transaction. * @param req Request. */ @@ -1483,22 +1477,27 @@ protected void finish( throw e; } catch (Throwable e) { - U.error(log, "Failed committing transaction [tx=" + tx + ']', e); + try { + // Mark transaction for invalidate. + tx.invalidate(true); - // Mark transaction for invalidate. - tx.invalidate(true); - tx.systemInvalidate(true); + tx.systemInvalidate(true); - try { - tx.rollbackRemoteTx(); + try { + tx.rollbackRemoteTx(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } + + tx.logTxFinishErrorSafe(log, true, e); + + if (e instanceof Error) + throw (Error)e; } - catch (Throwable e1) { - e.addSuppressed(e1); - U.error(log, "Failed to automatically rollback transaction: " + tx, e1); + finally { + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); } - - if (e instanceof Error) - throw (Error)e; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java index 775b61c0eeffe..7e042923ad48b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java @@ -558,7 +558,7 @@ protected GridCacheEntryEx entryEx(GridCacheContext cacheCtx, IgniteTxKey key, A /** {@inheritDoc} */ @SuppressWarnings({"CatchGenericClass"}) - @Override public final void userCommit() throws IgniteCheckedException { + @Override public void userCommit() throws IgniteCheckedException { TransactionState state = state(); if (state != COMMITTING) { @@ -590,7 +590,7 @@ protected GridCacheEntryEx entryEx(GridCacheContext cacheCtx, IgniteTxKey key, A WALPointer ptr = null; - Exception err = null; + IgniteCheckedException err = null; cctx.database().checkpointReadLock(); @@ -609,176 +609,175 @@ protected GridCacheEntryEx entryEx(GridCacheContext cacheCtx, IgniteTxKey key, A UUID nodeId = txEntry.nodeId() == null ? this.nodeId : txEntry.nodeId(); - try { - while (true) { - try { - GridCacheEntryEx cached = txEntry.cached(); + while (true) { + try { + GridCacheEntryEx cached = txEntry.cached(); - // Must try to evict near entries before committing from - // transaction manager to make sure locks are held. - if (!evictNearEntry(txEntry, false)) { - if (cacheCtx.isNear() && cacheCtx.dr().receiveEnabled()) { - cached.markObsolete(xidVer); + // Must try to evict near entries before committing from + // transaction manager to make sure locks are held. + if (!evictNearEntry(txEntry, false)) { + if (cacheCtx.isNear() && cacheCtx.dr().receiveEnabled()) { + cached.markObsolete(xidVer); - break; - } + break; + } - if (cached.detached()) - break; + if (cached.detached()) + break; - boolean updateNearCache = updateNearCache(cacheCtx, txEntry.key(), topVer); + boolean updateNearCache = updateNearCache(cacheCtx, txEntry.key(), topVer); - boolean metrics = true; + boolean metrics = true; - if (!updateNearCache && cacheCtx.isNear() && txEntry.locallyMapped()) - metrics = false; + if (!updateNearCache && cacheCtx.isNear() && txEntry.locallyMapped()) + metrics = false; - boolean evt = !isNearLocallyMapped(txEntry, false); + boolean evt = !isNearLocallyMapped(txEntry, false); - if (!F.isEmpty(txEntry.entryProcessors()) || !F.isEmpty(txEntry.filters())) - txEntry.cached().unswap(false); + if (!F.isEmpty(txEntry.entryProcessors()) || !F.isEmpty(txEntry.filters())) + txEntry.cached().unswap(false); - IgniteBiTuple res = applyTransformClosures(txEntry, - true, null); + IgniteBiTuple res = applyTransformClosures(txEntry, + true, null); - GridCacheVersion dhtVer = null; + GridCacheVersion dhtVer = null; - // For near local transactions we must record DHT version - // in order to keep near entries on backup nodes until - // backup remote transaction completes. - if (cacheCtx.isNear()) { - if (txEntry.op() == CREATE || txEntry.op() == UPDATE || - txEntry.op() == DELETE || txEntry.op() == TRANSFORM) - dhtVer = txEntry.dhtVersion(); + // For near local transactions we must record DHT version + // in order to keep near entries on backup nodes until + // backup remote transaction completes. + if (cacheCtx.isNear()) { + if (txEntry.op() == CREATE || txEntry.op() == UPDATE || + txEntry.op() == DELETE || txEntry.op() == TRANSFORM) + dhtVer = txEntry.dhtVersion(); - if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) && - txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) { - ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry); + if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) && + txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) { + ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry); - if (expiry != null) { - txEntry.cached().unswap(false); + if (expiry != null) { + txEntry.cached().unswap(false); - Duration duration = cached.hasValue() ? - expiry.getExpiryForUpdate() : expiry.getExpiryForCreation(); + Duration duration = cached.hasValue() ? + expiry.getExpiryForUpdate() : expiry.getExpiryForCreation(); - txEntry.ttl(CU.toTtl(duration)); - } + txEntry.ttl(CU.toTtl(duration)); } } + } - GridCacheOperation op = res.get1(); - CacheObject val = res.get2(); + GridCacheOperation op = res.get1(); + CacheObject val = res.get2(); - // Deal with conflicts. - GridCacheVersion explicitVer = txEntry.conflictVersion() != null ? - txEntry.conflictVersion() : writeVersion(); + // Deal with conflicts. + GridCacheVersion explicitVer = txEntry.conflictVersion() != null ? + txEntry.conflictVersion() : writeVersion(); - if ((op == CREATE || op == UPDATE) && - txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) { - ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry); + if ((op == CREATE || op == UPDATE) && + txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) { + ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry); - if (expiry != null) { - Duration duration = cached.hasValue() ? - expiry.getExpiryForUpdate() : expiry.getExpiryForCreation(); + if (expiry != null) { + Duration duration = cached.hasValue() ? + expiry.getExpiryForUpdate() : expiry.getExpiryForCreation(); - long ttl = CU.toTtl(duration); + long ttl = CU.toTtl(duration); - txEntry.ttl(ttl); + txEntry.ttl(ttl); - if (ttl == CU.TTL_ZERO) - op = DELETE; - } + if (ttl == CU.TTL_ZERO) + op = DELETE; } + } - boolean conflictNeedResolve = cacheCtx.conflictNeedResolve(); - - GridCacheVersionConflictContext conflictCtx = null; - - if (conflictNeedResolve) { - IgniteBiTuple conflictRes = - conflictResolve(op, txEntry, val, explicitVer, cached); + boolean conflictNeedResolve = cacheCtx.conflictNeedResolve(); - assert conflictRes != null; + GridCacheVersionConflictContext conflictCtx = null; - conflictCtx = conflictRes.get2(); + if (conflictNeedResolve) { + IgniteBiTuple conflictRes = + conflictResolve(op, txEntry, val, explicitVer, cached); - if (conflictCtx.isUseOld()) - op = NOOP; - else if (conflictCtx.isUseNew()) { - txEntry.ttl(conflictCtx.ttl()); - txEntry.conflictExpireTime(conflictCtx.expireTime()); - } - else { - assert conflictCtx.isMerge(); + assert conflictRes != null; - op = conflictRes.get1(); - val = txEntry.context().toCacheObject(conflictCtx.mergeValue()); - explicitVer = writeVersion(); + conflictCtx = conflictRes.get2(); - txEntry.ttl(conflictCtx.ttl()); - txEntry.conflictExpireTime(conflictCtx.expireTime()); - } + if (conflictCtx.isUseOld()) + op = NOOP; + else if (conflictCtx.isUseNew()) { + txEntry.ttl(conflictCtx.ttl()); + txEntry.conflictExpireTime(conflictCtx.expireTime()); } - else - // Nullify explicit version so that innerSet/innerRemove will work as usual. - explicitVer = null; + else { + assert conflictCtx.isMerge(); - if (sndTransformedVals || conflictNeedResolve) { - assert sndTransformedVals && cacheCtx.isReplicated() || conflictNeedResolve; + op = conflictRes.get1(); + val = txEntry.context().toCacheObject(conflictCtx.mergeValue()); + explicitVer = writeVersion(); - txEntry.value(val, true, false); - txEntry.op(op); - txEntry.entryProcessors(null); - txEntry.conflictVersion(explicitVer); + txEntry.ttl(conflictCtx.ttl()); + txEntry.conflictExpireTime(conflictCtx.expireTime()); } + } + else + // Nullify explicit version so that innerSet/innerRemove will work as usual. + explicitVer = null; - if (dhtVer == null) - dhtVer = explicitVer != null ? explicitVer : writeVersion(); + if (sndTransformedVals || conflictNeedResolve) { + assert sndTransformedVals && cacheCtx.isReplicated() || conflictNeedResolve; - if (op == CREATE || op == UPDATE) { - assert val != null : txEntry; + txEntry.value(val, true, false); + txEntry.op(op); + txEntry.entryProcessors(null); + txEntry.conflictVersion(explicitVer); + } - GridCacheUpdateTxResult updRes = cached.innerSet( - this, - eventNodeId(), - txEntry.nodeId(), - val, - false, - false, - txEntry.ttl(), - evt, - metrics, - txEntry.keepBinary(), - txEntry.hasOldValue(), - txEntry.oldValue(), - topVer, - null, - cached.detached() ? DR_NONE : drType, - txEntry.conflictExpireTime(), - cached.isNear() ? null : explicitVer, - CU.subjectId(this, cctx), - resolveTaskName(), - dhtVer, - null, - mvccSnapshot()); - - if (updRes.success()) { - txEntry.updateCounter(updRes.updateCounter()); - - GridLongList waitTxs = updRes.mvccWaitTransactions(); - - updateWaitTxs(waitTxs); - } + if (dhtVer == null) + dhtVer = explicitVer != null ? explicitVer : writeVersion(); + + if (op == CREATE || op == UPDATE) { + assert val != null : txEntry; + + GridCacheUpdateTxResult updRes = cached.innerSet( + this, + eventNodeId(), + txEntry.nodeId(), + val, + false, + false, + txEntry.ttl(), + evt, + metrics, + txEntry.keepBinary(), + txEntry.hasOldValue(), + txEntry.oldValue(), + topVer, + null, + cached.detached() ? DR_NONE : drType, + txEntry.conflictExpireTime(), + cached.isNear() ? null : explicitVer, + CU.subjectId(this, cctx), + resolveTaskName(), + dhtVer, + null, + mvccSnapshot()); + + if (updRes.success()) { + txEntry.updateCounter(updRes.updateCounter()); + + GridLongList waitTxs = updRes.mvccWaitTransactions(); + + updateWaitTxs(waitTxs); + } - if (updRes.loggedPointer() != null) - ptr = updRes.loggedPointer(); + if (updRes.loggedPointer() != null) + ptr = updRes.loggedPointer(); - if (updRes.success() && updateNearCache) { - final CacheObject val0 = val; - final boolean metrics0 = metrics; - final GridCacheVersion dhtVer0 = dhtVer; + if (updRes.success() && updateNearCache) { + final CacheObject val0 = val; + final boolean metrics0 = metrics; + final GridCacheVersion dhtVer0 = dhtVer; - updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerSet( + updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerSet( null, eventNodeId(), nodeId, @@ -801,46 +800,46 @@ else if (conflictCtx.isUseNew()) { dhtVer0, null, mvccSnapshot()) - ); - } + ); + } + } + else if (op == DELETE) { + GridCacheUpdateTxResult updRes = cached.innerRemove( + this, + eventNodeId(), + txEntry.nodeId(), + false, + evt, + metrics, + txEntry.keepBinary(), + txEntry.hasOldValue(), + txEntry.oldValue(), + topVer, + null, + cached.detached() ? DR_NONE : drType, + cached.isNear() ? null : explicitVer, + CU.subjectId(this, cctx), + resolveTaskName(), + dhtVer, + null, + mvccSnapshot()); + + if (updRes.success()) { + txEntry.updateCounter(updRes.updateCounter()); + + GridLongList waitTxs = updRes.mvccWaitTransactions(); + + updateWaitTxs(waitTxs); } - else if (op == DELETE) { - GridCacheUpdateTxResult updRes = cached.innerRemove( - this, - eventNodeId(), - txEntry.nodeId(), - false, - evt, - metrics, - txEntry.keepBinary(), - txEntry.hasOldValue(), - txEntry.oldValue(), - topVer, - null, - cached.detached() ? DR_NONE : drType, - cached.isNear() ? null : explicitVer, - CU.subjectId(this, cctx), - resolveTaskName(), - dhtVer, - null, - mvccSnapshot()); - - if (updRes.success()) { - txEntry.updateCounter(updRes.updateCounter()); - - GridLongList waitTxs = updRes.mvccWaitTransactions(); - - updateWaitTxs(waitTxs); - } - if (updRes.loggedPointer() != null) - ptr = updRes.loggedPointer(); + if (updRes.loggedPointer() != null) + ptr = updRes.loggedPointer(); - if (updRes.success() && updateNearCache) { - final boolean metrics0 = metrics; - final GridCacheVersion dhtVer0 = dhtVer; + if (updRes.success() && updateNearCache) { + final boolean metrics0 = metrics; + final GridCacheVersion dhtVer0 = dhtVer; - updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerRemove( + updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerRemove( null, eventNodeId(), nodeId, @@ -859,125 +858,78 @@ else if (op == DELETE) { dhtVer0, null, mvccSnapshot()) - ); - } + ); } - else if (op == RELOAD) { - cached.innerReload(); + } + else if (op == RELOAD) { + cached.innerReload(); - if (updateNearCache) - updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerReload()); + if (updateNearCache) + updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerReload()); + } + else if (op == READ) { + CacheGroupContext grp = cacheCtx.group(); + + if (grp.persistenceEnabled() && grp.walEnabled() && + cctx.snapshot().needTxReadLogging()) { + ptr = cctx.wal().log(new DataRecord(new DataEntry( + cacheCtx.cacheId(), + txEntry.key(), + val, + op, + nearXidVersion(), + writeVersion(), + 0, + txEntry.key().partition(), + txEntry.updateCounter()))); } - else if (op == READ) { - CacheGroupContext grp = cacheCtx.group(); - - if (grp.persistenceEnabled() && grp.walEnabled() && - cctx.snapshot().needTxReadLogging()) { - ptr = cctx.wal().log(new DataRecord(new DataEntry( - cacheCtx.cacheId(), - txEntry.key(), - val, - op, - nearXidVersion(), - writeVersion(), - 0, - txEntry.key().partition(), - txEntry.updateCounter()))); - } - ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry); + ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry); - if (expiry != null) { - Duration duration = expiry.getExpiryForAccess(); + if (expiry != null) { + Duration duration = expiry.getExpiryForAccess(); - if (duration != null) - cached.updateTtl(null, CU.toTtl(duration)); - } - - if (log.isDebugEnabled()) - log.debug("Ignoring READ entry when committing: " + txEntry); + if (duration != null) + cached.updateTtl(null, CU.toTtl(duration)); } - else { - assert ownsLock(txEntry.cached()) : - "Transaction does not own lock for group lock entry during commit [tx=" + - this + ", txEntry=" + txEntry + ']'; - if (conflictCtx == null || !conflictCtx.isUseOld()) { - if (txEntry.ttl() != CU.TTL_NOT_CHANGED) - cached.updateTtl(null, txEntry.ttl()); - } - - if (log.isDebugEnabled()) - log.debug("Ignoring NOOP entry when committing: " + txEntry); - } + if (log.isDebugEnabled()) + log.debug("Ignoring READ entry when committing: " + txEntry); } + else { + assert ownsLock(txEntry.cached()) : + "Transaction does not own lock for group lock entry during commit [tx=" + + this + ", txEntry=" + txEntry + ']'; + + if (conflictCtx == null || !conflictCtx.isUseOld()) { + if (txEntry.ttl() != CU.TTL_NOT_CHANGED) + cached.updateTtl(null, txEntry.ttl()); + } - // Check commit locks after set, to make sure that - // we are not changing obsolete entries. - // (innerSet and innerRemove will throw an exception - // if an entry is obsolete). - if (txEntry.op() != READ) - checkCommitLocks(cached); - - // Break out of while loop. - break; - } - // If entry cached within transaction got removed. - catch (GridCacheEntryRemovedException ignored) { - if (log.isDebugEnabled()) - log.debug("Got removed entry during transaction commit (will retry): " + txEntry); - - txEntry.cached(entryEx(cacheCtx, txEntry.txKey(), topologyVersion())); + if (log.isDebugEnabled()) + log.debug("Ignoring NOOP entry when committing: " + txEntry); + } } - } - } - catch (Throwable ex) { - // We are about to initiate transaction rollback when tx has started to committing. - // Need to remove version from committed list. - cctx.tm().removeCommittedTx(this); - boolean isNodeStopping = X.hasCause(ex, NodeStoppingException.class); - boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class); + // Check commit locks after set, to make sure that + // we are not changing obsolete entries. + // (innerSet and innerRemove will throw an exception + // if an entry is obsolete). + if (txEntry.op() != READ) + checkCommitLocks(cached); - IgniteCheckedException err0 = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " + - "(all transaction entries will be invalidated, however there was a window when " + - "entries for this transaction were visible to others): " + this, ex); - - if (isNodeStopping) { - U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this + - ", err=" + ex + ']'); - } - else if (hasInvalidEnvironmentIssue) { - U.warn(log, "Failed to commit transaction, node is in invalid state and will be stopped [tx=" + this + - ", err=" + ex + ']'); + // Break out of while loop. + break; } - else - U.error(log, "Commit failed.", err0); - - COMMIT_ERR_UPD.compareAndSet(this, null, err0); + // If entry cached within transaction got removed. + catch (GridCacheEntryRemovedException ignored) { + if (log.isDebugEnabled()) + log.debug("Got removed entry during transaction commit (will retry): " + txEntry); - state(UNKNOWN); - - if (hasInvalidEnvironmentIssue) - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex)); - else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or invalidation. - try { - // Courtesy to minimize damage. - uncommit(); - } - catch (Throwable ex1) { - U.error(log, "Failed to uncommit transaction: " + this, ex1); - - if (ex1 instanceof Error) - throw ex1; - } + txEntry.cached(entryEx(cacheCtx, txEntry.txKey(), topologyVersion())); } - - if (ex instanceof Error) - throw ex; - - throw err0; } + } // Apply cache sizes only for primary nodes. Update counters were applied on prepare state. @@ -988,11 +940,32 @@ else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or in if (ptr != null && !cctx.tm().logTxRecords()) cctx.wal().flush(ptr, false); } - catch (StorageException e) { - err = e; + catch (Throwable ex) { + // We are about to initiate transaction rollback when tx has started to committing. + // Need to remove version from committed list. + cctx.tm().removeCommittedTx(this); + + if (X.hasCause(ex, NodeStoppingException.class)) { + U.warn(log, "Failed to commit transaction, node is stopping [tx=" + CU.txString(this) + + ", err=" + ex + ']'); + + return; + } + + err = heuristicException(ex); + + COMMIT_ERR_UPD.compareAndSet(this, null, err); + + state(UNKNOWN); + + try { + uncommit(); + } + catch (Throwable e) { + err.addSuppressed(e); + } - throw new IgniteCheckedException("Failed to log transaction record " + - "(transaction will be rolled back): " + this, e); + throw err; } finally { cctx.database().checkpointReadUnlock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java index f31f0e93a78b1..ceeb4e1cacb2d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java @@ -24,6 +24,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.failure.FailureContext; import org.apache.ignite.failure.FailureHandler; +import org.apache.ignite.failure.NoOpFailureHandler; import org.apache.ignite.failure.StopNodeOrHaltFailureHandler; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.GridProcessorAdapter; @@ -78,6 +79,13 @@ public FailureProcessor(GridKernalContext ctx) { U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']'); } + /** + * @return @{code True} if a node will be stopped by current handler in near time. + */ + public boolean nodeStopping() { + return failureCtx != null && !(hnd instanceof NoOpFailureHandler); + } + /** * This method is used to initialize local failure handler if {@link IgniteConfiguration} don't contain configured one. * diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java index a7e6e8c72de42..f68ecd68e2591 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java @@ -41,7 +41,6 @@ import org.apache.ignite.internal.processors.timeout.GridSpiTimeoutObject; import org.apache.ignite.internal.util.IgniteExceptionRegistry; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiPredicate; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java index 89f0ca79b2a69..68104bfa721b2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java @@ -256,7 +256,7 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throw cfg.setIndexedTypes(idxTypes); if (cacheMode() == PARTITIONED) - cfg.setBackups(1); + cfg.setBackups(backups()); return cfg; } @@ -361,6 +361,13 @@ protected IgniteTransactions transactions() { return grid(0).transactions(); } + /** + * @return Backups. + */ + protected int backups() { + return 1; + } + /** * @param idx Index of grid. * @return Default cache. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java index 5f2e2edaccaf0..3e59c2a39c0ee 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java @@ -26,12 +26,10 @@ import java.util.TreeMap; import java.util.concurrent.Callable; import javax.cache.Cache; -import junit.framework.TestCase; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.IgniteTransactions; -import org.apache.ignite.Ignition; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.query.QueryCursor; @@ -47,6 +45,7 @@ import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.apache.ignite.spi.indexing.IndexingSpi; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -57,25 +56,19 @@ /** * Indexing Spi query only test */ -public class IndexingSpiQuerySelfTest extends TestCase { - public static final String CACHE_NAME = "test-cache"; +public class IndexingSpiQuerySelfTest extends GridCommonAbstractTest { + private IndexingSpi indexingSpi; /** {@inheritDoc} */ - @Override public void tearDown() throws Exception { - Ignition.stopAll(true); - } - - /** - * @return Configuration. - */ - protected IgniteConfiguration configuration() { - IgniteConfiguration cfg = new IgniteConfiguration(); + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); TcpDiscoverySpi disco = new TcpDiscoverySpi(); disco.setIpFinder(ipFinder); + cfg.setIndexingSpi(indexingSpi); cfg.setDiscoverySpi(disco); return cfg; @@ -86,17 +79,22 @@ protected CacheConfiguration cacheConfiguration(String cacheName) { return new CacheConfiguration<>(cacheName); } + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + /** * @throws Exception If failed. */ public void testSimpleIndexingSpi() throws Exception { - IgniteConfiguration cfg = configuration(); - - cfg.setIndexingSpi(new MyIndexingSpi()); + indexingSpi = new MyIndexingSpi(); - Ignite ignite = Ignition.start(cfg); + Ignite ignite = startGrid(0); - CacheConfiguration ccfg = cacheConfiguration(CACHE_NAME); + CacheConfiguration ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); IgniteCache cache = ignite.createCache(ccfg); @@ -113,13 +111,11 @@ public void testSimpleIndexingSpi() throws Exception { * @throws Exception If failed. */ public void testIndexingSpiWithDisabledQueryProcessor() throws Exception { - IgniteConfiguration cfg = configuration(); - - cfg.setIndexingSpi(new MyIndexingSpi()); + indexingSpi = new MyIndexingSpi(); - Ignite ignite = Ignition.start(cfg); + Ignite ignite = startGrid(0); - CacheConfiguration ccfg = cacheConfiguration(CACHE_NAME); + CacheConfiguration ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); IgniteCache cache = ignite.createCache(ccfg); @@ -136,13 +132,11 @@ public void testIndexingSpiWithDisabledQueryProcessor() throws Exception { * @throws Exception If failed. */ public void testBinaryIndexingSpi() throws Exception { - IgniteConfiguration cfg = configuration(); + indexingSpi = new MyBinaryIndexingSpi(); - cfg.setIndexingSpi(new MyBinaryIndexingSpi()); + Ignite ignite = startGrid(0); - Ignite ignite = Ignition.start(cfg); - - CacheConfiguration ccfg = cacheConfiguration(CACHE_NAME); + CacheConfiguration ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); IgniteCache cache = ignite.createCache(ccfg); @@ -168,13 +162,11 @@ public void testBinaryIndexingSpi() throws Exception { public void testNonBinaryIndexingSpi() throws Exception { System.setProperty(IgniteSystemProperties.IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI, "true"); - IgniteConfiguration cfg = configuration(); - - cfg.setIndexingSpi(new MyIndexingSpi()); + indexingSpi = new MyIndexingSpi(); - Ignite ignite = Ignition.start(cfg); + Ignite ignite = startGrid(0); - CacheConfiguration ccfg = cacheConfiguration(CACHE_NAME); + CacheConfiguration ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); IgniteCache cache = ignite.createCache(ccfg); @@ -198,13 +190,11 @@ public void testNonBinaryIndexingSpi() throws Exception { */ @SuppressWarnings("ThrowableResultOfMethodCallIgnored") public void testIndexingSpiFailure() throws Exception { - IgniteConfiguration cfg = configuration(); - - cfg.setIndexingSpi(new MyBrokenIndexingSpi()); + indexingSpi = new MyBrokenIndexingSpi(); - Ignite ignite = Ignition.start(cfg); + Ignite ignite = startGrid(0); - CacheConfiguration ccfg = cacheConfiguration(CACHE_NAME); + CacheConfiguration ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java index e59deed2eb767..ca80b13c21fe9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java @@ -17,6 +17,11 @@ package org.apache.ignite.internal.processors.cache.query; +import java.util.Collection; +import java.util.Iterator; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.Cache; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteTransactions; import org.apache.ignite.cache.CacheAtomicityMode; @@ -37,61 +42,64 @@ import org.apache.ignite.transactions.TransactionState; import org.jetbrains.annotations.Nullable; -import java.util.Collection; -import java.util.Iterator; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import javax.cache.Cache; - /** * Indexing Spi transactional query test */ public class IndexingSpiQueryTxSelfTest extends GridCacheAbstractSelfTest { - /** */ - private static AtomicInteger cnt; - /** {@inheritDoc} */ @Override protected int gridCount() { return 4; } - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - cnt = new AtomicInteger(); - - super.beforeTestsStarted(); - } - /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true); - if (cnt.getAndIncrement() == 0) - cfg.setClientMode(true); - else { - cfg.setIndexingSpi(new MyBrokenIndexingSpi()); + cfg.setClientMode("client".equals(igniteInstanceName)); + cfg.setIndexingSpi(new MyBrokenIndexingSpi()); - CacheConfiguration ccfg = cacheConfiguration(igniteInstanceName); - ccfg.setName("test-cache"); - ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + CacheConfiguration ccfg = cacheConfiguration(igniteInstanceName); + ccfg.setName(DEFAULT_CACHE_NAME); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - ccfg.setIndexedTypes(Integer.class, Integer.class); + ccfg.setIndexedTypes(Integer.class, Integer.class); + + cfg.setCacheConfiguration(ccfg); - cfg.setCacheConfiguration(ccfg); - } return cfg; } + /** */ + public void testIndexingSpiWithTxClient() throws Exception { + IgniteEx client = startGrid("client"); + + assertNotNull(client.cache(DEFAULT_CACHE_NAME)); + + doTestIndexingSpiWithTx(client, 0); + } + + /** */ + public void testIndexingSpiWithTxLocal() throws Exception { + IgniteEx ignite = (IgniteEx)primaryNode(0, DEFAULT_CACHE_NAME); + + doTestIndexingSpiWithTx(ignite, 0); + } + + /** */ + public void testIndexingSpiWithTxNotLocal() throws Exception { + IgniteEx ignite = (IgniteEx)primaryNode(0, DEFAULT_CACHE_NAME); + + doTestIndexingSpiWithTx(ignite, 1); + } + /** * @throws Exception If failed. */ @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - public void testIndexingSpiWithTx() throws Exception { - IgniteEx ignite = grid(0); - - final IgniteCache cache = ignite.cache("test-cache"); + private void doTestIndexingSpiWithTx(IgniteEx ignite, int key) throws Exception { + final IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); final IgniteTransactions txs = ignite.transactions(); @@ -104,7 +112,7 @@ public void testIndexingSpiWithTx() throws Exception { Transaction tx; try (Transaction tx0 = tx = txs.txStart(concurrency, isolation)) { - cache.put(1, 1); + cache.put(key, key); tx0.commit(); } @@ -114,6 +122,8 @@ public void testIndexingSpiWithTx() throws Exception { return null; } }, IgniteTxHeuristicCheckedException.class); + + checkFutures(); } } } @@ -135,7 +145,7 @@ private static class MyBrokenIndexingSpi extends IgniteSpiAdapter implements Ind /** {@inheritDoc} */ @Override public Iterator> query(@Nullable String cacheName, Collection params, @Nullable IndexingQueryFilter filters) throws IgniteSpiException { - return null; + return null; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java index fe27e6e119d24..01db74726405f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.Affinity; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.cluster.ClusterNode; @@ -40,10 +41,10 @@ import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; import org.apache.ignite.failure.FailureHandler; import org.apache.ignite.failure.StopNodeFailureHandler; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -73,7 +74,7 @@ public class AbstractTransactionIntergrityTest extends GridCommonAbstractTest { private static final int DFLT_ACCOUNTS_CNT = 32; /** Count of threads and caches. */ - private static final int DFLT_TX_THREADS_CNT = 20; + private static final int DFLT_TX_THREADS_CNT = Runtime.getRuntime().availableProcessors(); /** Count of nodes to start. */ private static final int DFLT_NODES_CNT = 3; @@ -126,16 +127,6 @@ protected boolean persistent() { return true; } - /** - * @return Flag enables cross-node transactions, - * when primary partitions participating in transaction spreaded across several cluster nodes. - */ - protected boolean crossNodeTransactions() { - // Commit error during cross node transactions breaks transaction integrity - // TODO: https://issues.apache.org/jira/browse/IGNITE-9086 - return false; - } - /** {@inheritDoc} */ @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { return new StopNodeFailureHandler(); @@ -148,14 +139,15 @@ protected boolean crossNodeTransactions() { cfg.setConsistentId(name); ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); - cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); - cfg.setLocalHost("127.0.0.1"); cfg.setDataStorageConfiguration(new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setMaxSize(256 * 1024 * 1024) - .setPersistenceEnabled(persistent())) - ); + .setPersistenceEnabled(persistent()) + .setMaxSize(50 * 1024 * 1024) + ) + .setWalSegmentSize(16 * 1024 * 1024) + .setPageSize(1024) + .setWalMode(WALMode.LOG_ONLY)); CacheConfiguration[] cacheConfigurations = new CacheConfiguration[txThreadsCount()]; @@ -178,6 +170,8 @@ protected boolean crossNodeTransactions() { cfg.setCacheConfiguration(cacheConfigurations); + cfg.setFailureDetectionTimeout(30_000); + return cfg; } @@ -219,8 +213,11 @@ protected boolean crossNodeTransactions() { /** * Test transfer amount. + * + * @param failoverScenario Scenario. + * @param colocatedAccounts {@code True} to use colocated on same primary node accounts. */ - public void doTestTransferAmount(FailoverScenario failoverScenario) throws Exception { + public void doTestTransferAmount(FailoverScenario failoverScenario, boolean colocatedAccounts) throws Exception { failoverScenario.beforeNodesStarted(); //given: started some nodes with client. @@ -230,26 +227,26 @@ public void doTestTransferAmount(FailoverScenario failoverScenario) throws Excep igniteClient.cluster().active(true); - int[] initAmount = new int[txThreadsCount()]; + int[] initAmounts = new int[txThreadsCount()]; completedTxs = new ConcurrentLinkedHashMap[txThreadsCount()]; //and: fill all accounts on all caches and calculate total amount for every cache. for (int cachePrefixIdx = 0; cachePrefixIdx < txThreadsCount(); cachePrefixIdx++) { IgniteCache cache = igniteClient.getOrCreateCache(cacheName(cachePrefixIdx)); - AtomicInteger coinsCounter = new AtomicInteger(); + AtomicInteger coinsCntr = new AtomicInteger(); try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { for (int accountId = 0; accountId < accountsCount(); accountId++) { - Set initialAmount = generateCoins(coinsCounter, 5); + Set initAmount = generateCoins(coinsCntr, 5); - cache.put(accountId, new AccountState(accountId, tx.xid(), initialAmount)); + cache.put(accountId, new AccountState(accountId, tx.xid(), initAmount)); } tx.commit(); } - initAmount[cachePrefixIdx] = coinsCounter.get(); + initAmounts[cachePrefixIdx] = coinsCntr.get(); completedTxs[cachePrefixIdx] = new ConcurrentLinkedHashMap(); } @@ -259,7 +256,8 @@ public void doTestTransferAmount(FailoverScenario failoverScenario) throws Excep ArrayList transferThreads = new ArrayList<>(); for (int i = 0; i < txThreadsCount(); i++) { - transferThreads.add(new TransferAmountTxThread(firstTransactionDone, igniteClient, cacheName(i), i)); + transferThreads.add(new TransferAmountTxThread(firstTransactionDone, + igniteClient, cacheName(i), i, colocatedAccounts)); transferThreads.get(i).start(); } @@ -268,13 +266,12 @@ public void doTestTransferAmount(FailoverScenario failoverScenario) throws Excep failoverScenario.afterFirstTransaction(); - for (Thread thread : transferThreads) { + for (Thread thread : transferThreads) thread.join(); - } failoverScenario.afterTransactionsFinished(); - consistencyCheck(initAmount); + consistencyCheck(initAmounts); } /** @@ -385,11 +382,11 @@ public AccountState addCoins(IgniteUuid txId, Set coinsToAdd) { /** * @param txId Transaction id. - * @param coinsToRemove Coins to remove from current account. + * @param coinsToRmv Coins to remove from current account. * @return Account state with removed coins. */ - public AccountState removeCoins(IgniteUuid txId, Set coinsToRemove) { - return new AccountState(accId, txId, Sets.difference(coins, coinsToRemove).immutableCopy()); + public AccountState removeCoins(IgniteUuid txId, Set coinsToRmv) { + return new AccountState(accId, txId, Sets.difference(coins, coinsToRmv).immutableCopy()); } /** {@inheritDoc} */ @@ -418,11 +415,11 @@ public AccountState removeCoins(IgniteUuid txId, Set coinsToRemove) { /** * @param coinsNum Coins number. */ - private Set generateCoins(AtomicInteger coinsCounter, int coinsNum) { + private Set generateCoins(AtomicInteger coinsCntr, int coinsNum) { Set res = new HashSet<>(); for (int i = 0; i < coinsNum; i++) - res.add(coinsCounter.incrementAndGet()); + res.add(coinsCntr.incrementAndGet()); return res; } @@ -479,23 +476,35 @@ public TxState(AccountState before1, AccountState before2, AccountState after1, private class TransferAmountTxThread extends Thread { /** */ private CountDownLatch firstTransactionLatch; + /** */ private IgniteEx ignite; + /** */ private String cacheName; + /** */ - private int txIndex; + private int workerIdx; + /** */ private Random random = new Random(); + /** */ + private final boolean colocatedAccounts; + /** * @param ignite Ignite. */ - private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final IgniteEx ignite, String cacheName, int txIndex) { + private TransferAmountTxThread(CountDownLatch firstTransactionLatch, + final IgniteEx ignite, + String cacheName, + int workerIdx, + boolean colocatedAccounts) { this.firstTransactionLatch = firstTransactionLatch; this.ignite = ignite; this.cacheName = cacheName; - this.txIndex = txIndex; + this.workerIdx = workerIdx; + this.colocatedAccounts = colocatedAccounts; } /** {@inheritDoc} */ @@ -514,7 +523,6 @@ private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final Ignit /** * @throws IgniteException if fails */ - @SuppressWarnings("unchecked") private void updateInTransaction(IgniteCache cache) throws IgniteException { int accIdFrom; int accIdTo; @@ -526,11 +534,16 @@ private void updateInTransaction(IgniteCache cache) throw if (accIdFrom == accIdTo) continue; - ClusterNode primaryForAccFrom = ignite.cachex(cacheName).affinity().mapKeyToNode(accIdFrom); - ClusterNode primaryForAccTo = ignite.cachex(cacheName).affinity().mapKeyToNode(accIdTo); + Affinity affinity = ignite.affinity(cacheName); + + ClusterNode primaryForAccFrom = affinity.mapKeyToNode(accIdFrom); + assertNotNull(primaryForAccFrom); + + ClusterNode primaryForAccTo = affinity.mapKeyToNode(accIdTo); + assertNotNull(primaryForAccTo); // Allows only transaction between accounts that primary on the same node if corresponding flag is enabled. - if (!crossNodeTransactions() && !primaryForAccFrom.id().equals(primaryForAccTo.id())) + if (colocatedAccounts && !primaryForAccFrom.id().equals(primaryForAccTo.id())) continue; break; @@ -541,7 +554,10 @@ private void updateInTransaction(IgniteCache cache) throw try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { acctFrom = cache.get(accIdFrom); + assertNotNull(acctFrom); + acctTo = cache.get(accIdTo); + assertNotNull(acctTo); Set coinsToTransfer = acctFrom.coinsToTransfer(random); @@ -553,23 +569,8 @@ private void updateInTransaction(IgniteCache cache) throw tx.commit(); - completedTxs[txIndex].put(tx.xid(), new TxState(acctFrom, acctTo, nextFrom, nextTo, coinsToTransfer)); - } - } - - /** - * @param curr current - * @return random value - */ - private long getNextAccountId(long curr) { - long randomVal; - - do { - randomVal = random.nextInt(accountsCount()); + completedTxs[workerIdx].put(tx.xid(), new TxState(acctFrom, acctTo, nextFrom, nextTo, coinsToTransfer)); } - while (curr == randomVal); - - return randomVal; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java index 3260607023a92..473eaf5ce97bc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java @@ -17,20 +17,26 @@ package org.apache.ignite.internal.processors.cache.transactions; +import java.util.Collection; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Supplier; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.Ignition; -import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler; import org.apache.ignite.internal.processors.cache.tree.SearchRow; import org.apache.ignite.testframework.GridTestUtils; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; + /** * Test cases that check transaction data integrity after transaction commit failed. */ @@ -45,81 +51,96 @@ public class TransactionIntegrityWithPrimaryIndexCorruptionTest extends Abstract super.afterTest(); } - /** {@inheritDoc} */ - @Override protected long getTestTimeout() { - return 60 * 1000L; + /** */ + public void testPrimaryIndexCorruptionDuringCommitPrimaryColocatedThrowsError() throws Exception { + doTestTransferAmount0(true, true, () -> new AssertionError("Test")); } - /** - * Throws a test {@link AssertionError} during tx commit from {@link BPlusTree} and checks after that data is consistent. - */ - public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode1() throws Exception { - doTestTransferAmount(new IndexCorruptionFailoverScenario( - true, - (hnd, tree) -> hnd instanceof BPlusTree.Search, - failoverPredicate(true, () -> new AssertionError("Test"))) - ); + /** */ + public void testPrimaryIndexCorruptionDuringCommitPrimaryColocatedThrowsUnchecked() throws Exception { + doTestTransferAmount0(true, true, () -> new RuntimeException("Test")); } - /** - * Throws a test {@link RuntimeException} during tx commit from {@link BPlusTree} and checks after that data is consistent. - */ - public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode2() throws Exception { - doTestTransferAmount(new IndexCorruptionFailoverScenario( - true, - (hnd, tree) -> hnd instanceof BPlusTree.Search, - failoverPredicate(true, () -> new RuntimeException("Test"))) - ); + /** */ + public void testPrimaryIndexCorruptionDuringCommitPrimaryColocatedThrowsChecked() throws Exception { + doTestTransferAmount0(true, true, () -> new IgniteCheckedException("Test")); } - /** - * Throws a test {@link AssertionError} during tx commit from {@link BPlusTree} and checks after that data is consistent. - */ - public void testPrimaryIndexCorruptionDuringCommitOnBackupNode() throws Exception { - doTestTransferAmount(new IndexCorruptionFailoverScenario( - true, - (hnd, tree) -> hnd instanceof BPlusTree.Search, - failoverPredicate(false, () -> new AssertionError("Test"))) - ); + /** */ + public void testPrimaryIndexCorruptionDuringCommitPrimaryNonColocatedThrowsError() throws Exception { + doTestTransferAmount0(false, true, () -> new AssertionError("Test")); } - /** - * Throws a test {@link IgniteCheckedException} during tx commit from {@link BPlusTree} and checks after that data is consistent. - */ - public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode3() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-9082"); + /** */ + public void testPrimaryIndexCorruptionDuringCommitPrimaryNonColocatedThrowsUnchecked() throws Exception { + doTestTransferAmount0(false, true, () -> new RuntimeException("Test")); + } - doTestTransferAmount(new IndexCorruptionFailoverScenario( - false, - (hnd, tree) -> hnd instanceof BPlusTree.Search, - failoverPredicate(true, () -> new IgniteCheckedException("Test"))) - ); + /** */ + public void testPrimaryIndexCorruptionDuringCommitPrimaryNonColocatedThrowsChecked() throws Exception { + doTestTransferAmount0(false, true, () -> new IgniteCheckedException("Test")); + } + + /** */ + public void testPrimaryIndexCorruptionDuringCommitBackupColocatedThrowsError() throws Exception { + doTestTransferAmount0(true, false, () -> new AssertionError("Test")); + } + + /** */ + public void testPrimaryIndexCorruptionDuringCommitBackupColocatedThrowsUnchecked() throws Exception { + doTestTransferAmount0(true, false, () -> new RuntimeException("Test")); + } + + /** */ + public void testPrimaryIndexCorruptionDuringCommitBackupColocatedThrowsChecked() throws Exception { + doTestTransferAmount0(true, false, () -> new IgniteCheckedException("Test")); + } + + /** */ + public void testPrimaryIndexCorruptionDuringCommitBackupNonColocatedThrowsError() throws Exception { + doTestTransferAmount0(false, false, () -> new AssertionError("Test")); + } + + /** */ + public void testPrimaryIndexCorruptionDuringCommitBackupNonColocatedThrowsUnchecked() throws Exception { + doTestTransferAmount0(false, false, () -> new RuntimeException("Test")); + } + + /** */ + public void testPrimaryIndexCorruptionDuringCommitBackupNonColocatedThrowsChecked() throws Exception { + doTestTransferAmount0(false, false, () -> new IgniteCheckedException("Test")); } /** * Creates failover predicate which generates error during transaction commmit. * - * @param failOnPrimary If {@code true} index should be failed on transaction primary node. + * @param failOnPrimary If {@code true} index should be failed on transaction primary node, otherwise on backup. * @param errorSupplier Supplier to create various errors. + * @param errorConsumer Consumer to track unexpected errors while committing. */ private BiFunction failoverPredicate( boolean failOnPrimary, - Supplier errorSupplier + Supplier errorSupplier, + Consumer errorConsumer ) { return (ignite, row) -> { - int cacheId = row.cacheId(); - int partId = row.key().partition(); - - final ClusterNode locNode = ignite.localNode(); - final AffinityTopologyVersion curTopVer = ignite.context().discovery().topologyVersionEx(); - - // Throw exception if current node is primary for given row. - return ignite.cachesx(c -> c.context().cacheId() == cacheId) - .stream() - .filter(c -> c.context().affinity().primaryByPartition(locNode, partId, curTopVer) == failOnPrimary) - .map(c -> errorSupplier.get()) - .findFirst() - .orElse(null); + try { + int cacheId = row.cacheId(); + int partId = row.key().partition(); + + GridDhtPartitionTopology top = ignite.context().cache().cacheGroup(cacheId).topology(); + + GridDhtLocalPartition part = top.localPartition(partId); + + assertTrue("Illegal partition state for mapped tx: " + part, part != null && part.state() == OWNING); + + return part.primary(top.readyTopologyVersion()) == failOnPrimary ? errorSupplier.get() : null; + } + catch (Throwable e) { + errorConsumer.accept(e); + + throw e; + } }; } @@ -130,68 +151,68 @@ class IndexCorruptionFailoverScenario implements FailoverScenario { /** Failed node index. */ static final int failedNodeIdx = 1; - /** Is node stopping expected after failover. */ - private final boolean nodeStoppingExpected; - - /** Predicate that will choose an instance of {@link BPlusTree} and page operation - * to make further failover in this tree using {@link #failoverPredicate}. */ - private final BiFunction treeCorruptionPredicate; + /** + * Predicate that will choose an instance of {@link BPlusTree} and page operation to make further failover in + * this tree using {@link #failoverPred}. + */ + private final BiFunction treeCorruptionPred; /** Function that may return error during row insertion into {@link BPlusTree}. */ - private final BiFunction failoverPredicate; + private final BiFunction failoverPred; /** - * @param nodeStoppingExpected Node stopping expected. - * @param treeCorruptionPredicate Tree corruption predicate. - * @param failoverPredicate Failover predicate. + * @param treeCorruptionPred Tree corruption predicate. + * @param failoverPred Failover predicate. */ IndexCorruptionFailoverScenario( - boolean nodeStoppingExpected, - BiFunction treeCorruptionPredicate, - BiFunction failoverPredicate + BiFunction treeCorruptionPred, + BiFunction failoverPred ) { - this.nodeStoppingExpected = nodeStoppingExpected; - this.treeCorruptionPredicate = treeCorruptionPredicate; - this.failoverPredicate = failoverPredicate; + this.treeCorruptionPred = treeCorruptionPred; + this.failoverPred = failoverPred; } /** {@inheritDoc} */ @Override public void beforeNodesStarted() { BPlusTree.pageHndWrapper = (tree, hnd) -> { - final IgniteEx locIgnite = (IgniteEx) Ignition.localIgnite(); + final IgniteEx locIgnite = (IgniteEx)Ignition.localIgnite(); - if (!locIgnite.name().endsWith(String.valueOf(failedNodeIdx))) + if (getTestIgniteInstanceIndex(locIgnite.name()) != failedNodeIdx) return hnd; - if (treeCorruptionPredicate.apply(hnd, tree)) { - log.info("Created corrupted tree handler for -> " + hnd + " " + tree); + if (treeCorruptionPred.apply(hnd, tree)) { + log.info("Created corrupted tree handler [nodeOrder=" + locIgnite.localNode().order() + ", hnd=" + hnd + + ", tree=" + tree + ']'); - PageHandler delegate = (PageHandler) hnd; + PageHandler delegate = (PageHandler)hnd; return new PageHandler() { - @Override public BPlusTree.Result run(int cacheId, long pageId, long page, long pageAddr, PageIO io, Boolean walPlc, BPlusTree.Get arg, int lvl) throws IgniteCheckedException { - log.info("Invoked " + " " + cacheId + " " + arg.toString() + " for BTree (" + corruptionEnabled + ") -> " + arg.row() + " / " + arg.row().getClass()); + @Override public BPlusTree.Result run(int cacheId, long pageId, long page, long pageAddr, PageIO io, + Boolean walPlc, BPlusTree.Get arg, int lvl) throws IgniteCheckedException { + log.info("Invoked [cachedId=" + cacheId + ", hnd=" + arg.toString() + + ", corruption=" + corruptionEnabled + ", row=" + arg.row() + ", rowCls=" + arg.row().getClass() + ']'); if (corruptionEnabled && (arg.row() instanceof SearchRow)) { - SearchRow row = (SearchRow) arg.row(); + SearchRow row = (SearchRow)arg.row(); // Store cacheId to search row explicitly, as it can be zero if there is one cache in a group. - Throwable res = failoverPredicate.apply(locIgnite, new SearchRow(cacheId, row.key())); + Throwable res = failoverPred.apply(locIgnite, new SearchRow(cacheId, row.key())); if (res != null) { if (res instanceof Error) - throw (Error) res; + throw (Error)res; else if (res instanceof RuntimeException) - throw (RuntimeException) res; + throw (RuntimeException)res; else if (res instanceof IgniteCheckedException) - throw (IgniteCheckedException) res; + throw (IgniteCheckedException)res; } } return delegate.run(cacheId, pageId, page, pageAddr, io, walPlc, arg, lvl); } - @Override public boolean releaseAfterWrite(int cacheId, long pageId, long page, long pageAddr, BPlusTree.Get g, int lvl) { + @Override public boolean releaseAfterWrite(int cacheId, long pageId, long page, long pageAddr, + BPlusTree.Get g, int lvl) { return g.canRelease(pageId, lvl); } }; @@ -212,27 +233,68 @@ else if (res instanceof IgniteCheckedException) // Disable index corruption. BPlusTree.pageHndWrapper = (tree, hnd) -> hnd; - if (nodeStoppingExpected) { - // Wait until node with corrupted index will left cluster. - GridTestUtils.waitForCondition(() -> { - try { - grid(failedNodeIdx); - } - catch (IgniteIllegalStateException e) { - return true; - } + // Wait until node with corrupted index will left cluster. + GridTestUtils.waitForCondition(() -> { + try { + grid(failedNodeIdx); + } + catch (IgniteIllegalStateException e) { + return true; + } - return false; - }, getTestTimeout()); + return false; + }, getTestTimeout()); - // Failed node should be stopped. - GridTestUtils.assertThrows(log, () -> grid(failedNodeIdx), IgniteIllegalStateException.class, ""); + // Failed node should be stopped. + GridTestUtils.assertThrows(log, () -> grid(failedNodeIdx), IgniteIllegalStateException.class, null); - // Re-start failed node. - startGrid(failedNodeIdx); + // Re-start failed node. + startGrid(failedNodeIdx); - awaitPartitionMapExchange(); - } + awaitPartitionMapExchange(); + } + } + + /** + * Test transfer amount with extended error recording. + * + * @param colocatedAccount Colocated account. + * @param failOnPrimary {@code True} if fail on primary, else on backup. + * @param supplier Fail reason supplier. + * @throws Exception If failover predicate execution is failed. + */ + private void doTestTransferAmount0(boolean colocatedAccount, boolean failOnPrimary, Supplier supplier) throws Exception { + ErrorTracker errTracker = new ErrorTracker(); + + doTestTransferAmount( + new IndexCorruptionFailoverScenario( + (hnd, tree) -> hnd instanceof BPlusTree.Search, + failoverPredicate(failOnPrimary, supplier, errTracker)), + colocatedAccount + ); + + for (Throwable throwable : errTracker.errors()) + log.error("Recorded error", throwable); + + if (!errTracker.errors().isEmpty()) + fail("Test run has error"); + } + + /** */ + private static class ErrorTracker implements Consumer { + /** Queue. */ + private final Queue q = new ConcurrentLinkedQueue<>(); + + /** {@inheritDoc} */ + @Override public void accept(Throwable throwable) { + q.add(throwable); + } + + /** + * @return Recorded errors. + */ + public Collection errors() { + return q; } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java index 25aae4b2a1b47..551335f2fee6f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java @@ -41,9 +41,7 @@ public class TransactionIntegrityWithSystemWorkerDeathTest extends AbstractTrans return false; } - /** - * - */ + /** */ public void testFailoverWithDiscoWorkerTermination() throws Exception { doTestTransferAmount(new FailoverScenario() { static final int failedNodeIdx = 1; @@ -83,7 +81,7 @@ public void testFailoverWithDiscoWorkerTermination() throws Exception { awaitPartitionMapExchange(); } - }); + }, true); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java new file mode 100644 index 0000000000000..881c6804255df --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.transactions; + +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteTransactions; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testsuites.IgniteIgnore; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; +import org.jetbrains.annotations.Nullable; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * Tests data consistency if transaction is failed due to heuristic exception on originating node. + */ +public class TxDataConsistencyOnCommitFailureTest extends GridCommonAbstractTest { + /** */ + public static final int KEY = 0; + + /** */ + public static final String CLIENT = "client"; + + /** */ + private int nodesCnt; + + /** */ + private int backups; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setClientMode(igniteInstanceName.startsWith(CLIENT)); + + cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME). + setCacheMode(CacheMode.PARTITIONED). + setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL). + setBackups(backups). + setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** */ + public void testCommitErrorOnNearNode2PC() throws Exception { + nodesCnt = 3; + + backups = 2; + + doTestCommitError(() -> { + try { + return startGrid("client"); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + /** */ + public void testCommitErrorOnNearNode1PC() throws Exception { + nodesCnt = 2; + + backups = 1; + + doTestCommitError(() -> { + try { + return startGrid("client"); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + /** */ + @IgniteIgnore(value = "https://issues.apache.org/jira/browse/IGNITE-9806", forceFailure = false) + public void testCommitErrorOnColocatedNode2PC() throws Exception { + nodesCnt = 3; + + backups = 2; + + doTestCommitError(() -> primaryNode(KEY, DEFAULT_CACHE_NAME)); + } + + /** + * @param factory Factory. + */ + private void doTestCommitError(Supplier factory) throws Exception { + Ignite crd = startGridsMultiThreaded(nodesCnt); + + crd.cache(DEFAULT_CACHE_NAME).put(KEY, KEY); + + Ignite ignite = factory.get(); + + if (ignite == null) + ignite = startGrid("client"); + + assertNotNull(ignite.cache(DEFAULT_CACHE_NAME)); + + injectMockedTxManager(ignite); + + checkKey(); + + IgniteTransactions transactions = ignite.transactions(); + + try(Transaction tx = transactions.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 0, 1)) { + assertNotNull(transactions.tx()); + + ignite.cache(DEFAULT_CACHE_NAME).put(KEY, KEY + 1); + + tx.commit(); + + fail(); + } + catch (Exception t) { + // No-op. + } + + checkKey(); + + checkFutures(); + } + + /** + * @param ignite Ignite. + */ + private void injectMockedTxManager(Ignite ignite) { + IgniteEx igniteEx = (IgniteEx)ignite; + + GridCacheSharedContext ctx = igniteEx.context().cache().context(); + + IgniteTxManager tm = ctx.tm(); + + IgniteTxManager mockTm = Mockito.spy(tm); + + MockGridNearTxLocal locTx = new MockGridNearTxLocal(ctx, false, false, false, GridIoPolicy.SYSTEM_POOL, + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 0, true, null, 1, null, 0, null); + + Mockito.doAnswer(new Answer() { + @Override public GridNearTxLocal answer(InvocationOnMock invocation) throws Throwable { + mockTm.onCreated(null, locTx); + + return locTx; + } + }).when(mockTm). + newTx(locTx.implicit(), locTx.implicitSingle(), null, locTx.concurrency(), + locTx.isolation(), locTx.timeout(), locTx.storeEnabled(), null, locTx.size(), locTx.label()); + + ctx.setTxManager(mockTm); + } + + /** */ + private void checkKey() { + for (Ignite ignite : G.allGrids()) { + if (!ignite.configuration().isClientMode()) + assertNotNull(ignite.cache(DEFAULT_CACHE_NAME).localPeek(KEY)); + } + } + + /** */ + private static class MockGridNearTxLocal extends GridNearTxLocal { + /** Empty constructor. */ + public MockGridNearTxLocal() { + } + + /** + * @param ctx Context. + * @param implicit Implicit. + * @param implicitSingle Implicit single. + * @param sys System. + * @param plc Policy. + * @param concurrency Concurrency. + * @param isolation Isolation. + * @param timeout Timeout. + * @param storeEnabled Store enabled. + * @param mvccOp Mvcc op. + * @param txSize Tx size. + * @param subjId Subj id. + * @param taskNameHash Task name hash. + * @param lb Label. + */ + public MockGridNearTxLocal(GridCacheSharedContext ctx, boolean implicit, boolean implicitSingle, boolean sys, + byte plc, TransactionConcurrency concurrency, TransactionIsolation isolation, long timeout, + boolean storeEnabled, Boolean mvccOp, int txSize, @Nullable UUID subjId, int taskNameHash, @Nullable String lb) { + super(ctx, implicit, implicitSingle, sys, plc, concurrency, isolation, timeout, storeEnabled, mvccOp, + txSize, subjId, taskNameHash, lb); + } + + /** {@inheritDoc} */ + @Override public void userCommit() throws IgniteCheckedException { + throw new IgniteCheckedException("Force failure"); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 7e98ec763e3aa..7091a0990d71c 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -1996,17 +1996,26 @@ protected void checkFutures() { final Collection> futs = ig.context().cache().context().mvcc().activeFutures(); - for (GridCacheFuture fut : futs) - log.info("Waiting for future: " + fut); + boolean hasFutures = false; - assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty()); + for (GridCacheFuture fut : futs) { + if (!fut.isDone()) { + log.error("Expecting no active future [node=" + ig.localNode().id() + ", fut=" + fut + ']'); + + hasFutures = true; + } + } + + if (hasFutures) + fail("Some mvcc futures are not finished"); Collection txs = ig.context().cache().context().tm().activeTransactions(); for (IgniteInternalTx tx : txs) - log.info("Waiting for tx: " + tx); + log.error("Expecting no active transaction [node=" + ig.localNode().id() + ", tx=" + tx + ']'); - assertTrue("Expecting no active transactions: node=" + ig.localNode().id(), txs.isEmpty()); + if (!txs.isEmpty()) + fail("Some transaction are not finished"); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index 386b17bacae2e..7dba46114ccbc 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest; +import org.apache.ignite.internal.processors.cache.transactions.TxDataConsistencyOnCommitFailureTest; import org.apache.ignite.testframework.junits.GridAbstractTest; /** @@ -49,6 +50,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class); + suite.addTestSuite(TxDataConsistencyOnCommitFailureTest.class); + return suite; } } From f605905758a7d64eb6b5a4e6dcd72a886809d487 Mon Sep 17 00:00:00 2001 From: macrergate Date: Tue, 16 Oct 2018 18:17:10 +0300 Subject: [PATCH 035/403] IGNITE-9430 Add ability to override all caches's "rebalanceThrottle" option via JVM node option - Fixes #4944. Signed-off-by: Ivan Rakov (cherry picked from commit 1db895545b303b8945b9b8ade237e17ccd434f5c) --- .../org/apache/ignite/IgniteSystemProperties.java | 6 ++++++ .../dht/preloader/GridDhtPartitionSupplier.java | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 692cfe489a8d9..9f349547dae37 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1002,6 +1002,12 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_REUSE_MEMORY_ON_DEACTIVATE = "IGNITE_REUSE_MEMORY_ON_DEACTIVATE"; + /** + * System property to override {@link CacheConfiguration#rebalanceThrottle} configuration property for all caches. + * {@code 0} by default, which means that override is disabled. + */ + public static final String IGNITE_REBALANCE_THROTTLE_OVERRIDE = "IGNITE_REBALANCE_THROTTLE_OVERRIDE"; + /** * Timeout for waiting schema update if schema was not found for last accepted version. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java index 835910e36387c..92547667ca39e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java @@ -27,6 +27,7 @@ import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; @@ -45,6 +46,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.T3; +import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; @@ -71,6 +73,10 @@ class GridDhtPartitionSupplier { /** Supply context map. T3: nodeId, topicId, topVer. */ private final Map, SupplyContext> scMap = new HashMap<>(); + /** Override for rebalance throttle. */ + private long rebalanceThrottleOverride = + IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_REBALANCE_THROTTLE_OVERRIDE, 0); + /** * @param grp Cache group. */ @@ -82,6 +88,9 @@ class GridDhtPartitionSupplier { log = grp.shared().logger(getClass()); top = grp.topology(); + + if (rebalanceThrottleOverride > 0) + LT.info(log, "Using rebalance throttle override: " + rebalanceThrottleOverride); } /** @@ -511,7 +520,9 @@ private boolean reply( grp.shared().io().sendOrderedMessage(demander, demandMsg.topic(), supplyMsg, grp.ioPolicy(), demandMsg.timeout()); // Throttle preloading. - if (grp.config().getRebalanceThrottle() > 0) + if (rebalanceThrottleOverride > 0) + U.sleep(rebalanceThrottleOverride); + else if (grp.config().getRebalanceThrottle() > 0) U.sleep(grp.config().getRebalanceThrottle()); return true; From e8e2e9514db04831616dbbcd9af7cbf4e2e9b463 Mon Sep 17 00:00:00 2001 From: Sergey Kosarev Date: Wed, 24 Oct 2018 16:04:05 +0300 Subject: [PATCH 036/403] fix compilation --- .../main/java/org/apache/ignite/IgniteSystemProperties.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 9f349547dae37..692cfe489a8d9 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1002,12 +1002,6 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_REUSE_MEMORY_ON_DEACTIVATE = "IGNITE_REUSE_MEMORY_ON_DEACTIVATE"; - /** - * System property to override {@link CacheConfiguration#rebalanceThrottle} configuration property for all caches. - * {@code 0} by default, which means that override is disabled. - */ - public static final String IGNITE_REBALANCE_THROTTLE_OVERRIDE = "IGNITE_REBALANCE_THROTTLE_OVERRIDE"; - /** * Timeout for waiting schema update if schema was not found for last accepted version. */ From 46f0a8d46d4157fcb0fdaf72f14683300e52b9b3 Mon Sep 17 00:00:00 2001 From: Denis Mekhanikov Date: Wed, 24 Oct 2018 12:24:46 +0300 Subject: [PATCH 037/403] IGNITE-9899 Notify CQ filter on backups - Fixes #5011. Signed-off-by: Alexey Goncharuk (cherry picked from commit 0d7535340b61df0c327fd6afe9d3519a248ee350) --- .../processors/cache/GridCacheMapEntry.java | 13 +++---- ...ontinuousQueryAsyncFailoverTxSelfTest.java | 5 --- ...acheContinuousQueryFailoverTxSelfTest.java | 5 --- ...tinuousQueryOperationFromCallbackTest.java | 36 +++++++++++++++++++ 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index c753296b528da..347e2607344a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1462,7 +1462,7 @@ else if (res.resultType() == ResultType.LOCKED) { boolean internal = isInternal() || !context().userCache(); Map lsnrCol = - notifyContinuousQueries(tx) ? + notifyContinuousQueries() ? cctx.continuousQueries().updateListeners(internal, false) : null; if (startVer && (retval || intercept || lsnrCol != null)) @@ -1695,7 +1695,7 @@ protected Object keyValue(boolean cpy) { boolean internal = isInternal() || !context().userCache(); Map lsnrCol = - notifyContinuousQueries(tx) ? + notifyContinuousQueries() ? cctx.continuousQueries().updateListeners(internal, false) : null; if (startVer && (retval || intercept || lsnrCol != null)) @@ -2949,13 +2949,10 @@ protected final void update(@Nullable CacheObject val, long expireTime, long ttl } /** - * @param tx Transaction. - * @return {@code True} if should notify continuous query manager. + * @return {@code True} if should notify continuous query manager on updates of this entry. */ - private boolean notifyContinuousQueries(@Nullable IgniteInternalTx tx) { - return cctx.isLocal() || - cctx.isReplicated() || - (!isNear() && !(tx != null && tx.onePhaseCommit() && !tx.local())); + private boolean notifyContinuousQueries() { + return !isNear(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java index 0417022cdbeb7..8f0bd0e0d4633 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverTxSelfTest.java @@ -41,9 +41,4 @@ public class CacheContinuousQueryAsyncFailoverTxSelfTest extends CacheContinuous @Override protected boolean asyncCallback() { return true; } - - /** {@inheritDoc} */ - @Override public void testNoEventLossOnTopologyChange() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-4015"); - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java index cd916e4217001..789a105e02b05 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxSelfTest.java @@ -36,9 +36,4 @@ public class CacheContinuousQueryFailoverTxSelfTest extends CacheContinuousQuery @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; } - - /** {@inheritDoc} */ - @Override public void testNoEventLossOnTopologyChange() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-4015"); - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java index 0540b43207a6f..a9e3d439d5305 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryOperationFromCallbackTest.java @@ -133,6 +133,42 @@ public class CacheContinuousQueryOperationFromCallbackTest extends GridCommonAbs filterCbCntr.set(0); } + /** + * @throws Exception If failed. + */ + public void testAtomicOneBackup() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, ATOMIC, FULL_SYNC); + + doTest(ccfg, true); + } + + /** + * @throws Exception If failed. + */ + public void testTxOneBackupFilter() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL, FULL_SYNC); + + doTest(ccfg, false); + } + + /** + * @throws Exception If failed. + */ + public void testTxOneBackupFilterPrimary() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL, PRIMARY_SYNC); + + doTest(ccfg, false); + } + + /** + * @throws Exception If failed. + */ + public void testTxOneBackup() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL, FULL_SYNC); + + doTest(ccfg, true); + } + /** * @throws Exception If failed. */ From 3ecddf45b94c2c58541318d6c6734a6e789a9d86 Mon Sep 17 00:00:00 2001 From: Stanislav Lukyanov Date: Mon, 22 Oct 2018 15:08:10 +0300 Subject: [PATCH 038/403] IGNITE-9900: Upgrade annotations.jar to a new version. - Fixes #5004. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit ebaedf9) --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index eeceb18e7c1ef..e260118bcf7c0 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -80,7 +80,7 @@ 1.0.0_1 1.0.1 1.0.0 - 13.0 + 16.0.3 9.4.11.v20180605 1.13 1.1.1 From 8421fa822762884664587ce230d580a362160f07 Mon Sep 17 00:00:00 2001 From: Andrei Aleksandrov Date: Wed, 24 Oct 2018 12:01:00 +0300 Subject: [PATCH 039/403] IGNITE-9914 Use correct subject ID for TaskEvents for remote clients - Fixes #5030. Signed-off-by: Alexey Goncharuk (cherry picked from commit 5e989bd) --- .../internal/TaskEventSubjectIdSelfTest.java | 9 +++-- .../processors/task/GridTaskProcessor.java | 5 ++- .../internal/GridJobSubjectIdSelfTest.java | 40 +++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/TaskEventSubjectIdSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/TaskEventSubjectIdSelfTest.java index 46aaa6ba387b6..ee6e9f101c757 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/TaskEventSubjectIdSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/TaskEventSubjectIdSelfTest.java @@ -308,6 +308,9 @@ public void testClosure() throws Exception { } /** + * Events for class tasks that was started from external clients should contain + * client subject id instead of the node where it was started. This test checks it. + * * @throws Exception If failed. */ public void testClient() throws Exception { @@ -328,7 +331,7 @@ public void testClient() throws Exception { assert evt != null; assertEquals(EVT_TASK_STARTED, evt.type()); - assertEquals(nodeId, evt.subjectId()); + assertEquals(client.id(), evt.subjectId()); assert it.hasNext(); @@ -337,7 +340,7 @@ public void testClient() throws Exception { assert evt != null; assertEquals(EVT_TASK_REDUCED, evt.type()); - assertEquals(nodeId, evt.subjectId()); + assertEquals(client.id(), evt.subjectId()); assert it.hasNext(); @@ -346,7 +349,7 @@ public void testClient() throws Exception { assert evt != null; assertEquals(EVT_TASK_FINISHED, evt.type()); - assertEquals(nodeId, evt.subjectId()); + assertEquals(client.id(), evt.subjectId()); assert !it.hasNext(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index 9007472b034f5..e3759fe3965fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -679,7 +679,10 @@ else if (task != null) { top = nodes != null ? F.nodeIds(nodes) : null; } - UUID subjId = getThreadContext(TC_SUBJ_ID); + UUID subjId = (UUID)map.get(TC_SUBJ_ID); + + if (subjId == null) + subjId = getThreadContext(TC_SUBJ_ID); if (subjId == null) subjId = ctx.localNodeId(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobSubjectIdSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobSubjectIdSelfTest.java index 07720675d379e..95a792697ecfe 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/GridJobSubjectIdSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobSubjectIdSelfTest.java @@ -37,6 +37,8 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBJ_ID; + /** * Test job subject ID propagation. */ @@ -59,6 +61,7 @@ public class GridJobSubjectIdSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { node1 = startGrid(1); + node2 = startGrid(2); } @@ -67,7 +70,14 @@ public class GridJobSubjectIdSelfTest extends GridCommonAbstractTest { stopAllGrids(); node1 = null; + node2 = null; + + evtSubjId = null; + + taskSubjId = null; + + jobSubjId = null; } /** @@ -91,9 +101,39 @@ public void testJobSubjectId() throws Exception { node1.compute().execute(new Task(node2.cluster().localNode().id()), null); assertEquals(taskSubjId, jobSubjId); + assertEquals(taskSubjId, evtSubjId); } + /** + * Test job subject ID propagation in case if was changed. + * + * @throws Exception If failed. + */ + public void testModifiedSubjectId() throws Exception { + node1.events().localListen(new IgnitePredicate() { + @Override public boolean apply(Event evt) { + JobEvent evt0 = (JobEvent)evt; + + assert evtSubjId == null; + + evtSubjId = evt0.taskSubjectId(); + + return false; + } + }, EventType.EVT_JOB_STARTED); + + UUID uuid = new UUID(100, 100); + + ((IgniteEx) node1).context().task().setThreadContextIfNotNull(TC_SUBJ_ID, uuid); + + ((IgniteEx) node1).context().task().execute(new Task(node1.cluster().localNode().id()), null).get(); + + assertEquals(uuid, jobSubjId); + + assertEquals(uuid, evtSubjId); + } + /** * Task class. */ From 22ca91055eb016e758d1c40742561fa067ad7f83 Mon Sep 17 00:00:00 2001 From: Andrei Aleksandrov Date: Wed, 24 Oct 2018 12:39:19 +0300 Subject: [PATCH 040/403] IGNITE-9881 Added events for management tasks - Fixes #5006. Signed-off-by: Alexey Goncharuk (cherry picked from commit ff29b67) --- .../org/apache/ignite/events/EventType.java | 10 + .../managers/deployment/GridDeployment.java | 16 ++ .../processors/task/GridTaskProcessor.java | 20 ++ .../task/GridVisorManagementTask.java | 38 ++++ .../visor/baseline/VisorBaselineTask.java | 2 + .../VisorBinaryMetadataCollectorTask.java | 2 + .../cache/VisorCacheAffinityNodeTask.java | 2 + .../visor/cache/VisorCacheClearTask.java | 2 + .../visor/cache/VisorCacheLoadTask.java | 2 + .../cache/VisorCacheLostPartitionsTask.java | 2 + .../VisorCacheLostPartitionsTaskArg.java | 10 +- .../visor/cache/VisorCacheModifyTask.java | 2 + .../visor/cache/VisorCacheModifyTaskArg.java | 6 + .../visor/cache/VisorCacheRebalanceTask.java | 2 + .../VisorCacheResetLostPartitionsTask.java | 2 + .../VisorCacheResetLostPartitionsTaskArg.java | 8 + .../visor/cache/VisorCacheStartTask.java | 2 + .../visor/cache/VisorCacheStopTask.java | 2 + .../VisorComputeCancelSessionsTask.java | 2 + .../compute/VisorComputeResetMetricsTask.java | 2 + .../visor/debug/VisorThreadDumpTask.java | 2 + .../visor/igfs/VisorIgfsFormatTask.java | 2 + .../igfs/VisorIgfsProfilerClearTask.java | 2 + .../visor/igfs/VisorIgfsResetMetricsTask.java | 2 + .../visor/log/VisorLogSearchTask.java | 2 + .../misc/VisorChangeGridActiveStateTask.java | 2 + .../internal/visor/node/VisorNodeGcTask.java | 2 + .../visor/node/VisorNodePingTask.java | 2 + .../visor/node/VisorNodeRestartTask.java | 2 + .../visor/node/VisorNodeStopTask.java | 2 + .../visor/query/VisorQueryCancelTask.java | 2 + .../query/VisorQueryResetMetricsTask.java | 2 + .../internal/visor/query/VisorQueryTask.java | 2 + .../visor/service/VisorCancelServiceTask.java | 2 + .../TestManagementVisorMultiNodeTask.java | 66 ++++++ .../TestManagementVisorOneNodeTask.java | 66 ++++++ .../TestNotManagementVisorMultiNodeTask.java | 64 ++++++ .../TestNotManagementVisorOneNodeTask.java | 64 ++++++ .../VisorManagementEventSelfTest.java | 192 ++++++++++++++++++ .../IgniteComputeGridTestSuite.java | 3 + 40 files changed, 616 insertions(+), 1 deletion(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridVisorManagementTask.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorMultiNodeTask.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorOneNodeTask.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorMultiNodeTask.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorOneNodeTask.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/VisorManagementEventSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/events/EventType.java b/modules/core/src/main/java/org/apache/ignite/events/EventType.java index 485e5671e6423..97017d638dd0a 100644 --- a/modules/core/src/main/java/org/apache/ignite/events/EventType.java +++ b/modules/core/src/main/java/org/apache/ignite/events/EventType.java @@ -244,6 +244,16 @@ public interface EventType { */ public static final int EVT_TASK_REDUCED = 25; + /** + * Built-in event type: Visor or Web Console management task started. + *

+ * NOTE: all types in range from 1 to 1000 are reserved for + * internal Ignite events and should not be used by user-defined events. + * + * @see TaskEvent + */ + public static final int EVT_MANAGEMENT_TASK_STARTED = 26; + /** * Built-in event type: non-task class deployed. *

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeployment.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeployment.java index c3efc592b1347..3450aa5195c4a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeployment.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeployment.java @@ -35,6 +35,7 @@ import org.apache.ignite.compute.ComputeTask; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.GridLeanSet; import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter; import org.apache.ignite.internal.util.lang.GridPeerDeployAware; @@ -45,6 +46,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteUuid; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.concurrent.ConcurrentHashMap; @@ -383,6 +385,20 @@ public boolean internalTask(@Nullable ComputeTask task, Class taskCls) { return res; } + /** + * Checks whether task class is annotated with {@link GridVisorManagementTask}. + * + * @param task Task. + * @param taskCls Task class. + * @return {@code True} if task is internal. + */ + @SuppressWarnings("unchecked") + public boolean visorManagementTask(@Nullable ComputeTask task, @NotNull Class taskCls) { + return annotation(task instanceof GridPeerDeployAware ? + ((GridPeerDeployAware)task).deployClass() : taskCls, + GridVisorManagementTask.class) != null; + } + /** * @param cls Class to create new instance of (using default constructor). * @return New instance. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index e3759fe3965fc..f4d3ccfd0ae2a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -70,6 +70,7 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; @@ -80,6 +81,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.events.EventType.EVT_TASK_SESSION_ATTR_SET; +import static org.apache.ignite.events.EventType.EVT_MANAGEMENT_TASK_STARTED; import static org.apache.ignite.internal.GridTopic.TOPIC_JOB_SIBLINGS; import static org.apache.ignite.internal.GridTopic.TOPIC_TASK; import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL; @@ -745,6 +747,24 @@ else if (task != null) { assert taskWorker0 == null : "Session ID is not unique: " + sesId; + if (ctx.event().isRecordable(EVT_MANAGEMENT_TASK_STARTED) && dep.visorManagementTask(task, taskCls)) { + VisorTaskArgument visorTaskArgument = (VisorTaskArgument)arg; + + Event evt = new TaskEvent( + ctx.discovery().localNode(), + visorTaskArgument != null && visorTaskArgument.getArgument() != null + ? visorTaskArgument.getArgument().toString() : "[]", + EVT_MANAGEMENT_TASK_STARTED, + ses.getId(), + taskCls == null ? null : taskCls.getSimpleName(), + "VisorManagementTask", + false, + subjId + ); + + ctx.event().record(evt); + } + if (!ctx.clientDisconnected()) { if (dep.annotation(taskCls, ComputeTaskMapAsync.class) != null) { try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridVisorManagementTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridVisorManagementTask.java new file mode 100644 index 0000000000000..09c87bf017964 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridVisorManagementTask.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.task; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that annotated task is a visor task that was invoked by user. They can be handled by event listeners. + * + * This annotation intended for internal use only. + */ +@Documented +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface GridVisorManagementTask { + // No-op. +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java index 3c00452c83a48..be5fcca5e85b5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java @@ -27,6 +27,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.cluster.IgniteClusterEx; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; @@ -37,6 +38,7 @@ * Task that will collect information about baseline topology and can change its state. */ @GridInternal +@GridVisorManagementTask public class VisorBaselineTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/binary/VisorBinaryMetadataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/binary/VisorBinaryMetadataCollectorTask.java index bf072e22485de..c62c658e529ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/binary/VisorBinaryMetadataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/binary/VisorBinaryMetadataCollectorTask.java @@ -20,6 +20,7 @@ import org.apache.ignite.IgniteBinary; import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -29,6 +30,7 @@ * Task that collects binary metadata. */ @GridInternal +@GridVisorManagementTask public class VisorBinaryMetadataCollectorTask extends VisorOneNodeTask { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheAffinityNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheAffinityNodeTask.java index 447cc11d7d387..439b8d3edcd7b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheAffinityNodeTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheAffinityNodeTask.java @@ -21,6 +21,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -30,6 +31,7 @@ * Task that will find affinity node for key. */ @GridInternal +@GridVisorManagementTask public class VisorCacheAffinityNodeTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java index bdfc9eb183a1f..68142b97aad60 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java @@ -21,6 +21,7 @@ import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.compute.ComputeJobContext; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -32,6 +33,7 @@ * Task that clears specified caches on specified node. */ @GridInternal +@GridVisorManagementTask public class VisorCacheClearTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLoadTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLoadTask.java index 7b25ae4e84cf4..5d1bccd06f3bf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLoadTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLoadTask.java @@ -26,6 +26,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; @@ -35,6 +36,7 @@ * Task to loads caches. */ @GridInternal +@GridVisorManagementTask public class VisorCacheLoadTask extends VisorOneNodeTask> { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java index 9785b9cf96921..daa4af7bec8ae 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java @@ -24,6 +24,7 @@ import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -32,6 +33,7 @@ * Collect list of lost partitions. */ @GridInternal +@GridVisorManagementTask public class VisorCacheLostPartitionsTask extends VisorOneNodeTask { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java index d6404bfdeb5ec..3ccaeaa3ba179 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java @@ -35,6 +35,9 @@ public class VisorCacheLostPartitionsTaskArg extends VisorDataTransferObject { /** List of cache names. */ private List cacheNames; + /** Created for toString method because Collection can't be printed. */ + private String modifiedCaches; + /** * Default constructor. */ @@ -47,6 +50,9 @@ public VisorCacheLostPartitionsTaskArg() { */ public VisorCacheLostPartitionsTaskArg(List cacheNames) { this.cacheNames = cacheNames; + + if (cacheNames != null) + modifiedCaches = cacheNames.toString(); } /** @@ -59,11 +65,13 @@ public List getCacheNames() { /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeCollection(out, cacheNames); + U.writeString(out, modifiedCaches); } /** {@inheritDoc} */ - @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { cacheNames = U.readList(in); + modifiedCaches = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java index 62e7ac6449cf3..f68efa8622e1b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java @@ -21,6 +21,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -31,6 +32,7 @@ * Task that modify value in specified cache. */ @GridInternal +@GridVisorManagementTask public class VisorCacheModifyTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java index bef73ed5fa508..3948a2f086672 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java @@ -43,6 +43,9 @@ public class VisorCacheModifyTaskArg extends VisorDataTransferObject { /** Specified value. */ private Object val; + /** Created for toString method because Object can't be printed. */ + private String modifiedValues; + /** * Default constructor. */ @@ -61,6 +64,7 @@ public VisorCacheModifyTaskArg(String cacheName, VisorModifyCacheMode mode, Obje this.mode = mode; this.key = key; this.val = val; + this.modifiedValues = "[Key=" + key + ", Value=" + val + "]"; } /** @@ -97,6 +101,7 @@ public Object getValue() { U.writeEnum(out, mode); out.writeObject(key); out.writeObject(val); + U.writeString(out, modifiedValues); } /** {@inheritDoc} */ @@ -105,6 +110,7 @@ public Object getValue() { mode = VisorModifyCacheMode.fromOrdinal(in.readByte()); key = in.readObject(); val = in.readObject(); + modifiedValues = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheRebalanceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheRebalanceTask.java index 87a2ce6713c22..9c3fd37e32bb6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheRebalanceTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheRebalanceTask.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; @@ -32,6 +33,7 @@ * Pre-loads caches. Made callable just to conform common pattern. */ @GridInternal +@GridVisorManagementTask public class VisorCacheRebalanceTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java index 2ad88ee75b7b6..aa90d3e52a77d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.visor.cache; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -26,6 +27,7 @@ * Reset lost partitions for caches. */ @GridInternal +@GridVisorManagementTask public class VisorCacheResetLostPartitionsTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java index 2f365c82d6fba..5f607304cb8d8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java @@ -35,6 +35,9 @@ public class VisorCacheResetLostPartitionsTaskArg extends VisorDataTransferObjec /** List of cache names. */ private List cacheNames; + /** Created for toString method because Collection can't be printed. */ + private String modifiedCaches; + /** * Default constructor. */ @@ -47,6 +50,9 @@ public VisorCacheResetLostPartitionsTaskArg() { */ public VisorCacheResetLostPartitionsTaskArg(List cacheNames) { this.cacheNames = cacheNames; + + if(cacheNames != null) + modifiedCaches = cacheNames.toString(); } /** @@ -59,11 +65,13 @@ public List getCacheNames() { /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeCollection(out, cacheNames); + U.writeString(out, modifiedCaches); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { cacheNames = U.readList(in); + modifiedCaches = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStartTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStartTask.java index 5f6337be0147e..7663827b53e6b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStartTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStartTask.java @@ -29,6 +29,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; @@ -40,6 +41,7 @@ * Task that start cache or near cache with specified configuration. */ @GridInternal +@GridVisorManagementTask public class VisorCacheStartTask extends VisorMultiNodeTask, Void> { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStopTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStopTask.java index df95c5ea73488..b76fc3df40315 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStopTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheStopTask.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.HashSet; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; @@ -29,6 +30,7 @@ * Task that stop specified caches on specified node. */ @GridInternal +@GridVisorManagementTask public class VisorCacheStopTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeCancelSessionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeCancelSessionsTask.java index 6cd683ca42318..8b29a400e19da 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeCancelSessionsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeCancelSessionsTask.java @@ -24,6 +24,7 @@ import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.compute.ComputeTaskFuture; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -34,6 +35,7 @@ * Cancels given tasks sessions. */ @GridInternal +@GridVisorManagementTask public class VisorComputeCancelSessionsTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeResetMetricsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeResetMetricsTask.java index eefeb746240d8..ac3e18b4a0d93 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeResetMetricsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorComputeResetMetricsTask.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.visor.compute; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -26,6 +27,7 @@ * Reset compute grid metrics task. */ @GridInternal +@GridVisorManagementTask public class VisorComputeResetMetricsTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadDumpTask.java index 9c6aa603a10b5..f064028abcf29 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadDumpTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadDumpTask.java @@ -20,6 +20,7 @@ import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; @@ -29,6 +30,7 @@ * Creates thread dump. */ @GridInternal +@GridVisorManagementTask public class VisorThreadDumpTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsFormatTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsFormatTask.java index e4685b73bcf6d..340abea3b0505 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsFormatTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsFormatTask.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -27,6 +28,7 @@ * Format IGFS instance. */ @GridInternal +@GridVisorManagementTask public class VisorIgfsFormatTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java index c730840ab621b..dc1a30ada5060 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java @@ -28,6 +28,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteFileSystem; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; @@ -39,6 +40,7 @@ * Remove all IGFS profiler logs. */ @GridInternal +@GridVisorManagementTask public class VisorIgfsProfilerClearTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsResetMetricsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsResetMetricsTask.java index ee90edf3bc78e..b1eaddad9d683 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsResetMetricsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsResetMetricsTask.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -27,6 +28,7 @@ * Resets IGFS metrics. */ @GridInternal +@GridVisorManagementTask public class VisorIgfsResetMetricsTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java index e4d2882568d1a..04ee29cda70bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/log/VisorLogSearchTask.java @@ -30,6 +30,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.io.GridReversedLinesFileReader; import org.apache.ignite.internal.util.lang.GridTuple3; import org.apache.ignite.internal.util.typedef.internal.S; @@ -48,6 +49,7 @@ * Search text matching in logs */ @GridInternal +@GridVisorManagementTask public class VisorLogSearchTask extends VisorMultiNodeTask> { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorChangeGridActiveStateTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorChangeGridActiveStateTask.java index bde4d6ccfc806..86aee0343202b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorChangeGridActiveStateTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorChangeGridActiveStateTask.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.visor.misc; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -26,6 +27,7 @@ * Task for changing grid active state. */ @GridInternal +@GridVisorManagementTask public class VisorChangeGridActiveStateTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeGcTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeGcTask.java index bdc1104cc5dcf..1e0ca4717d4f7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeGcTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeGcTask.java @@ -25,6 +25,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; @@ -34,6 +35,7 @@ * Task to run gc on nodes. */ @GridInternal +@GridVisorManagementTask public class VisorNodeGcTask extends VisorMultiNodeTask, VisorNodeGcTaskResult> { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodePingTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodePingTask.java index ee2e968ffb11f..656d12aee3d52 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodePingTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodePingTask.java @@ -21,6 +21,7 @@ import org.apache.ignite.cluster.ClusterTopologyException; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -30,6 +31,7 @@ * Ping other node. */ @GridInternal +@GridVisorManagementTask public class VisorNodePingTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeRestartTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeRestartTask.java index bb293710e8051..693665a672451 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeRestartTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeRestartTask.java @@ -21,6 +21,7 @@ import org.apache.ignite.Ignition; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; @@ -30,6 +31,7 @@ * Restarts nodes. */ @GridInternal +@GridVisorManagementTask public class VisorNodeRestartTask extends VisorMultiNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeStopTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeStopTask.java index 16bf5d6288cd0..6581e44e352c3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeStopTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeStopTask.java @@ -21,6 +21,7 @@ import org.apache.ignite.Ignition; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; @@ -30,6 +31,7 @@ * Stops nodes. */ @GridInternal +@GridVisorManagementTask public class VisorNodeStopTask extends VisorMultiNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryCancelTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryCancelTask.java index 207b690bc9bd5..32dc93cabe3d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryCancelTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryCancelTask.java @@ -22,6 +22,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; import org.jetbrains.annotations.Nullable; @@ -30,6 +31,7 @@ * Task to cancel queries. */ @GridInternal +@GridVisorManagementTask public class VisorQueryCancelTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetMetricsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetMetricsTask.java index 1d807f1aeb6e5..a339f8d790474 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetMetricsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryResetMetricsTask.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -27,6 +28,7 @@ * Reset compute grid query metrics. */ @GridInternal +@GridVisorManagementTask public class VisorQueryResetMetricsTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java index b5af1b04b5151..97ee83e128f7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -44,6 +45,7 @@ * Task for execute SQL fields query and get first page of results. */ @GridInternal +@GridVisorManagementTask public class VisorQueryTask extends VisorOneNodeTask> { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java index 53c3bb3fa8914..1bb7ecde4e39f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorCancelServiceTask.java @@ -19,6 +19,7 @@ import org.apache.ignite.IgniteServices; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; @@ -27,6 +28,7 @@ * Task for cancel services with specified name. */ @GridInternal +@GridVisorManagementTask public class VisorCancelServiceTask extends VisorOneNodeTask { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorMultiNodeTask.java b/modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorMultiNodeTask.java new file mode 100644 index 0000000000000..beccd799043dd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorMultiNodeTask.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal; + +import java.util.List; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorMultiNodeTask; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +@GridVisorManagementTask +public class TestManagementVisorMultiNodeTask extends VisorMultiNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorValidMultiNodeJob job(VisorTaskArgument arg) { + return new VisorValidMultiNodeJob(arg, debug); + } + + /** {@inheritDoc} */ + @Nullable @Override protected Object reduce0(List results) { + return null; + } + + /** + * Valid Management multi node visor job. + */ + private static class VisorValidMultiNodeJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Argument. + * @param debug Debug flag. + */ + protected VisorValidMultiNodeJob(VisorTaskArgument arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Object run(VisorTaskArgument arg) { + return null; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorOneNodeTask.java b/modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorOneNodeTask.java new file mode 100644 index 0000000000000..f7479af08b19b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestManagementVisorOneNodeTask.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal; + +import java.util.List; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.processors.task.GridVisorManagementTask; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +@GridVisorManagementTask +public class TestManagementVisorOneNodeTask extends VisorOneNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorValidOneNodeJob job(VisorTaskArgument arg) { + return new VisorValidOneNodeJob(arg, debug); + } + + /** {@inheritDoc} */ + @Nullable @Override protected Object reduce0(List results) { + return null; + } + + /** + * Valid Management one node visor job. + */ + private static class VisorValidOneNodeJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Argument. + * @param debug Debug flag. + */ + protected VisorValidOneNodeJob(VisorTaskArgument arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Object run(VisorTaskArgument arg) { + return null; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorMultiNodeTask.java b/modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorMultiNodeTask.java new file mode 100644 index 0000000000000..d8f279363385f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorMultiNodeTask.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal; + +import java.util.List; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorMultiNodeTask; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +public class TestNotManagementVisorMultiNodeTask extends VisorMultiNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorNotManagementMultiNodeJob job(VisorTaskArgument arg) { + return new VisorNotManagementMultiNodeJob(arg, debug); + } + + /** {@inheritDoc} */ + @Nullable @Override protected Object reduce0(List results) { + return null; + } + + /** + * Not management multi node visor job. + */ + private static class VisorNotManagementMultiNodeJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Argument. + * @param debug Debug flag. + */ + protected VisorNotManagementMultiNodeJob(VisorTaskArgument arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Object run(VisorTaskArgument arg) { + return null; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorOneNodeTask.java b/modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorOneNodeTask.java new file mode 100644 index 0000000000000..ea47ee09be730 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestNotManagementVisorOneNodeTask.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal; + +import java.util.List; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +public class TestNotManagementVisorOneNodeTask extends VisorOneNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorNotManagementOneNodeJob job(VisorTaskArgument arg) { + return new VisorNotManagementOneNodeJob(arg, debug); + } + + /** {@inheritDoc} */ + @Nullable @Override protected Object reduce0(List results) { + return null; + } + + /** + * Not management one node visor job. + */ + private static class VisorNotManagementOneNodeJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Argument. + * @param debug Debug flag. + */ + protected VisorNotManagementOneNodeJob(VisorTaskArgument arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Object run(VisorTaskArgument arg) { + return null; + } + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/VisorManagementEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/VisorManagementEventSelfTest.java new file mode 100644 index 0000000000000..1e8e0ca89c324 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/VisorManagementEventSelfTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.compute.ComputeTask; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.EventType; +import org.apache.ignite.events.TaskEvent; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.events.EventType.EVT_MANAGEMENT_TASK_STARTED;; + +/** + * + */ +public class VisorManagementEventSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = new IgniteConfiguration(); + + // Enable visor management events. + cfg.setIncludeEventTypes( + EVT_MANAGEMENT_TASK_STARTED + ); + + cfg.setCacheConfiguration( + new CacheConfiguration() + .setName("TEST") + .setIndexedTypes(Integer.class, Integer.class) + .setStatisticsEnabled(true) + ); + + TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(); + + List addrs = Arrays.asList("127.0.0.1:47500..47502"); + + ipFinder.setAddresses(addrs); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(ipFinder); + + cfg.setDiscoverySpi(discoSpi); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** + * Current test case start valid one node visor task that has GridVisorManagementTask annotation. + * No exceptions are expected. + * + * @throws Exception If failed. + */ + public void testManagementOneNodeVisorTask() throws Exception { + IgniteEx ignite = startGrid(0); + + doTestVisorTask(TestManagementVisorOneNodeTask.class, new VisorTaskArgument(), ignite); + } + + /** + * Current test case start valid multi node visor task that has GridVisorManagementTask annotation. + * No exceptions are expected. + * + * @throws Exception If failed. + */ + public void testManagementMultiNodeVisorTask() throws Exception { + IgniteEx ignite = startGrid(0); + + doTestVisorTask(TestManagementVisorMultiNodeTask.class, new VisorTaskArgument(), ignite); + } + + /** + * Current test case start one node visor task that has not GridVisorManagementTask annotation. + * No exceptions are expected. + * + * @throws Exception If failed. + */ + public void testNotManagementOneNodeVisorTask() throws Exception { + IgniteEx ignite = startGrid(0); + + doTestNotManagementVisorTask(TestNotManagementVisorOneNodeTask.class, new VisorTaskArgument(), ignite); + } + + /** + * Current test case start multi node visor task that has not GridVisorManagementTask annotation. + * No exceptions are expected. + * + * @throws Exception If failed. + */ + public void testNotManagementMultiNodeVisorTask() throws Exception { + IgniteEx ignite = startGrid(0); + + doTestNotManagementVisorTask(TestNotManagementVisorMultiNodeTask.class, new VisorTaskArgument(), ignite); + } + + /** + * @param cls class of the task. + * @param arg argument. + * @param ignite instance of Ignite. + * + * @throws Exception If failed. + */ + private void doTestVisorTask( + Class, R>> cls, T arg, IgniteEx ignite) throws Exception + { + final AtomicReference evt = new AtomicReference<>(); + + final CountDownLatch evtLatch = new CountDownLatch(1); + + ignite.events().localListen(new IgnitePredicate() { + @Override public boolean apply(TaskEvent e) { + evt.set(e); + + evtLatch.countDown(); + + return false; + } + }, EventType.EVT_MANAGEMENT_TASK_STARTED); + + for (ClusterNode node : ignite.cluster().forServers().nodes()) + ignite.compute().executeAsync(cls, new VisorTaskArgument<>(node.id(), arg, true)); + + assertTrue(evtLatch.await(10000, TimeUnit.MILLISECONDS)); + + assertNotNull(evt.get()); + } + + /** + * @param cls class of the task. + * @param arg argument. + * @param ignite instance of Ignite. + * + * @throws Exception If failed. + */ + private void doTestNotManagementVisorTask( + Class, R>> cls, T arg, IgniteEx ignite) throws Exception + { + final AtomicReference evt = new AtomicReference<>(); + + final CountDownLatch evtLatch = new CountDownLatch(1); + + ignite.events().localListen(new IgnitePredicate() { + @Override public boolean apply(TaskEvent e) { + evt.set(e); + + evtLatch.countDown(); + + return false; + } + }, EventType.EVT_MANAGEMENT_TASK_STARTED); + + for (ClusterNode node : ignite.cluster().forServers().nodes()) + ignite.compute().executeAsync(cls, new VisorTaskArgument<>(node.id(), arg, true)); + + assertFalse(evtLatch.await(10000, TimeUnit.MILLISECONDS)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java index 2e5708d5743ea..2771864e1ff6e 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java @@ -77,6 +77,7 @@ import org.apache.ignite.internal.IgniteExplicitImplicitDeploymentSelfTest; import org.apache.ignite.internal.IgniteRoundRobinErrorAfterClientReconnectTest; import org.apache.ignite.internal.TaskNodeRestartTest; +import org.apache.ignite.internal.VisorManagementEventSelfTest; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManagerSelfTest; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointTaskSelfTest; import org.apache.ignite.internal.managers.communication.GridCommunicationManagerListenersSelfTest; @@ -178,6 +179,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteComputeJobOneThreadTest.class); + suite.addTestSuite(VisorManagementEventSelfTest.class); + return suite; } } From 4c327672a26689b93f1f7686c1eb584c83b4559e Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Tue, 23 Oct 2018 12:33:24 +0300 Subject: [PATCH 041/403] IGNITE-8879 Fix blinking baseline node sometimes unable to connect to cluster - Fixes #4893. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit 3f7109ff52bdfc3157987b2230216cc982392eff) --- .../GridDhtPartitionsExchangeFuture.java | 19 ++++---- .../GridCacheDatabaseSharedManager.java | 22 ++++----- .../IgniteCacheDatabaseSharedManager.java | 6 ++- .../IgniteMetaStorageBasicTest.java | 45 +++++++++++++++++++ 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 0c2cbe26c54f5..63d36354327a6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -871,16 +871,13 @@ else if (msg instanceof WalStateAbstractMessage) * @throws IgniteCheckedException If failed. */ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedException { - if (isLocalNodeNotInBaseline()) { + boolean baselineNode = isLocalNodeInBaseline(); + + if (!baselineNode) { cctx.exchange().exchangerBlockingSectionBegin(); try { cctx.cache().cleanupCachesDirectories(); - - cctx.database().cleanupCheckpointDirectory(); - - if (cctx.wal() != null) - cctx.wal().cleanupWalDirectories(); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -916,7 +913,7 @@ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedExce cctx.exchange().exchangerBlockingSectionBegin(); try { - cctx.database().readCheckpointAndRestoreMemory(startDescs); + cctx.database().readCheckpointAndRestoreMemory(startDescs, !baselineNode); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -950,12 +947,12 @@ private void ensureClientCachesStarted() { } /** - * @return {@code true} if local node is not in baseline and {@code false} otherwise. + * @return {@code true} if local node is in baseline and {@code false} otherwise. */ - private boolean isLocalNodeNotInBaseline() { + private boolean isLocalNodeInBaseline() { BaselineTopology topology = cctx.discovery().discoCache().state().baselineTopology(); - return topology!= null && !topology.consistentIds().contains(cctx.localNode().consistentId()); + return topology != null && topology.consistentIds().contains(cctx.localNode().consistentId()); } /** @@ -1098,7 +1095,7 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) { cctx.exchange().exchangerBlockingSectionBegin(); try { - cctx.database().readCheckpointAndRestoreMemory(startDescs); + cctx.database().readCheckpointAndRestoreMemory(startDescs, false); } finally { cctx.exchange().exchangerBlockingSectionEnd(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 7dbb80a4b9ea6..3843ce1fe86e4 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -645,7 +645,7 @@ private void readMetastore() throws IgniteCheckedException { checkpointReadLock(); try { - restoreMemory(status, true, storePageMem); + restoreMemory(status, true, storePageMem, false); metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true); @@ -790,7 +790,8 @@ private void unRegistrateMetricsMBean() { /** {@inheritDoc} */ @Override public void readCheckpointAndRestoreMemory( - List cachesToStart + List cachesToStart, + boolean restoreMetastorageOnly ) throws IgniteCheckedException { assert !cctx.localNode().isClient(); @@ -819,7 +820,7 @@ private void unRegistrateMetricsMBean() { (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME) ); - WALPointer restore = restoreMemory(status); + WALPointer restore = restoreMemory(status, restoreMetastorageOnly, (PageMemoryEx) metaStorage.pageMemory(), true); if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " + @@ -1948,26 +1949,19 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC } } - /** - * @param status Checkpoint status. - * @throws IgniteCheckedException If failed. - * @throws StorageException In case I/O error occurred during operations with storage. - */ - @Nullable private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException { - return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory()); - } - /** * @param status Checkpoint status. * @param metastoreOnly If {@code True} restores Metastorage only. * @param storePageMem Metastore page memory. + * @param finalizeCp If {@code True}, finalizes checkpoint on recovery. * @throws IgniteCheckedException If failed. * @throws StorageException In case I/O error occurred during operations with storage. */ @Nullable private WALPointer restoreMemory( CheckpointStatus status, boolean metastoreOnly, - PageMemoryEx storePageMem + PageMemoryEx storePageMem, + boolean finalizeCp ) throws IgniteCheckedException { assert !metastoreOnly || storePageMem != null; @@ -2127,7 +2121,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC } } - if (metastoreOnly) + if (!finalizeCp) return null; WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 28ce085608d7c..db6b987bc7b68 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -649,9 +649,13 @@ public DataStorageMetrics persistentStoreMetrics() { /** * @param cachesToStart Started caches. + * @param restoreMetastorageOnly Apply updates only for metastorage. * @throws IgniteCheckedException If failed. */ - public void readCheckpointAndRestoreMemory(List cachesToStart) throws IgniteCheckedException { + public void readCheckpointAndRestoreMemory( + List cachesToStart, + boolean restoreMetastorageOnly + ) throws IgniteCheckedException { // No-op. } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java index 18375150d06d4..e5a53fac1833b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java @@ -17,12 +17,14 @@ package org.apache.ignite.internal.processors.cache.persistence.metastorage; import java.io.Serializable; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -106,6 +108,34 @@ public void testMetaStorageMassivePutUpdateRestart() throws Exception { verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX); } + /** + * @throws Exception If fails. + */ + public void testRecoveryOfMetastorageWhenNodeNotInBaseline() throws Exception { + IgniteEx ig0 = startGrid(0); + + ig0.cluster().active(true); + + final byte KEYS_CNT = 100; + final String KEY_PREFIX = "test.key."; + final String NEW_VAL_PREFIX = "new.val."; + final String UPDATED_VAL_PREFIX = "updated.val."; + + startGrid(1); + + // Disable checkpoints in order to check whether recovery works. + forceCheckpoint(grid(1)); + disableCheckpoints(grid(1)); + + loadKeys(grid(1), KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX); + + stopGrid(1, true); + + startGrid(1); + + verifyKeys(grid(1), KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX); + } + /** */ private void loadKeys(IgniteEx ig, byte keysCnt, @@ -144,4 +174,19 @@ private void verifyKeys(IgniteEx ig, Assert.assertEquals(valPrefix + i, val); } } + + /** + * Disable checkpoints on a specific node. + * + * @param node Ignite node.h + * @throws IgniteCheckedException If failed. + */ + private void disableCheckpoints(Ignite node) throws IgniteCheckedException { + assert !node.cluster().localNode().isClient(); + + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)node).context() + .cache().context().database(); + + dbMgr.enableCheckpoints(false).get(); + } } From 4b18beccc82193b91494f2868712e38dd6ff5113 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Thu, 25 Oct 2018 11:23:58 +0700 Subject: [PATCH 042/403] IGNITE-9853 control.sh: Added output of cache configurations. Fixes #4956. (cherry picked from commit a61e4a394af649a146a3f43489181db6fd991967) --- .../internal/commandline/CommandHandler.java | 615 ++++++++++++++++-- .../internal/commandline/OutputFormat.java | 66 ++ .../commandline/cache/CacheArguments.java | 27 + .../processors/cache/verify/CacheInfo.java | 71 +- .../visor/cache/VisorCacheConfiguration.java | 5 +- .../VisorCacheConfigurationCollectorJob.java | 23 +- ...sorCacheConfigurationCollectorTaskArg.java | 32 +- .../visor/verify/VisorViewCacheTaskArg.java | 35 +- .../ignite/util/GridCommandHandlerTest.java | 70 +- 9 files changed, 826 insertions(+), 118 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index c484018e0e568..35981b5fd7e1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -21,14 +21,15 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.UUID; -import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; @@ -65,6 +66,7 @@ import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.internal.visor.baseline.VisorBaselineNode; @@ -72,11 +74,20 @@ import org.apache.ignite.internal.visor.baseline.VisorBaselineTask; import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg; import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult; +import org.apache.ignite.internal.visor.cache.VisorCacheAffinityConfiguration; +import org.apache.ignite.internal.visor.cache.VisorCacheConfiguration; +import org.apache.ignite.internal.visor.cache.VisorCacheConfigurationCollectorTask; +import org.apache.ignite.internal.visor.cache.VisorCacheConfigurationCollectorTaskArg; +import org.apache.ignite.internal.visor.cache.VisorCacheEvictionConfiguration; +import org.apache.ignite.internal.visor.cache.VisorCacheNearConfiguration; +import org.apache.ignite.internal.visor.cache.VisorCacheRebalanceConfiguration; +import org.apache.ignite.internal.visor.cache.VisorCacheStoreConfiguration; import org.apache.ignite.internal.visor.misc.VisorClusterNode; import org.apache.ignite.internal.visor.misc.VisorWalTask; import org.apache.ignite.internal.visor.misc.VisorWalTaskArg; import org.apache.ignite.internal.visor.misc.VisorWalTaskOperation; import org.apache.ignite.internal.visor.misc.VisorWalTaskResult; +import org.apache.ignite.internal.visor.query.VisorQueryConfiguration; import org.apache.ignite.internal.visor.tx.VisorTxInfo; import org.apache.ignite.internal.visor.tx.VisorTxOperation; import org.apache.ignite.internal.visor.tx.VisorTxProjection; @@ -98,6 +109,7 @@ import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult; import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg; import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult; +import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd; import org.apache.ignite.internal.visor.verify.VisorViewCacheTask; import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg; import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult; @@ -115,11 +127,21 @@ import static org.apache.ignite.internal.commandline.Command.STATE; import static org.apache.ignite.internal.commandline.Command.TX; import static org.apache.ignite.internal.commandline.Command.WAL; +import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE; +import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.CONTENTION; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.DISTRIBUTION; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.HELP; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.IDLE_VERIFY; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.LIST; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.RESET_LOST_PARTITIONS; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.VALIDATE_INDEXES; import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD; import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT; import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE; import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.SET; import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.VERSION; +import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.CACHES; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.GROUPS; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.SEQ; @@ -127,9 +149,6 @@ * Class that execute several commands passed via command line. */ public class CommandHandler { - /** Logger. */ - private static final Logger log = Logger.getLogger(CommandHandler.class.getName()); - /** */ static final String DFLT_HOST = "127.0.0.1"; @@ -279,6 +298,27 @@ public class CommandHandler { /** */ private static final String TX_KILL = "kill"; + /** */ + private static final String OUTPUT_FORMAT = "--output-format"; + + /** */ + private static final String CONFIG = "--config"; + + /** Utility name. */ + private static final String UTILITY_NAME = "control.sh"; + + /** Common options. */ + private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"), op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT")); + + /** Utility name with common options. */ + private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS); + + /** Indent for help output. */ + private static final String INDENT = " "; + + /** */ + private static final String NULL = "null"; + /** */ private Iterator argsIt; @@ -303,6 +343,54 @@ private void log(String s) { System.out.println(s); } + /** + * Adds indent to begin of input string. + * + * @param s Input string. + * @return Indented string. + */ + private static String i(String s) { + return i(s, 1); + } + + /** + * Adds specified indents to begin of input string. + * + * @param s Input string. + * @param indentCnt Number of indents. + * @return Indented string. + */ + private static String i(String s, int indentCnt) { + assert indentCnt >= 0; + + switch (indentCnt) { + case 0: + return s; + + case 1: + return INDENT + s; + + default: + SB sb = new SB(s.length() + indentCnt * INDENT.length()); + + for (int i = 0; i < indentCnt; i++) + sb.a(INDENT); + + return sb.a(s).toString(); + } + + } + + /** + * Format and output specified string to console. + * + * @param format A format string as described in Format string syntax. + * @param args Arguments referenced by the format specifiers in the format string. + */ + private void log(String format, Object... args) { + System.out.printf(format, args); + } + /** * Provides a prompt, then reads a single line of text from the console. * @@ -319,7 +407,7 @@ private String readLine(String prompt) { * Output empty line. */ private void nl() { - System.out.println(""); + System.out.println(); } /** @@ -617,19 +705,21 @@ private void cache(GridClient client, CacheArguments cacheArgs) throws Throwable * */ private void printCacheHelp() { - log("--cache subcommand allows to do the following operations:"); - - usage(" Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]"); - usage(" Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]"); - usage(" Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [--dump] [--skipZeros] [cache1,...,cacheN]"); - usage(" Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]"); - usage(" Collect partition distribution information:", CACHE, " distribution nodeId|null [cacheName1,...,cacheNameN] [--user-attributes attributeName1[,...,attributeNameN]]"); - usage(" Reset lost partitions:", CACHE, " reset_lost_partitions cacheName1[,...,cacheNameN]"); - - log(" If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes."); - log(" Another commands where [nodeId] is optional will run on a random server node."); - log(" checkFirst numeric parameter for validate_indexes specifies number of first K keys to be validated."); - log(" checkThrough numeric parameter for validate_indexes allows to check each Kth key."); + log(i("The '" + CACHE.text() + " subcommand' is used to get information about and perform actions with caches. The command has the following syntax:")); + nl(); + log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + CACHE.text() + "[subcommand] ")); + nl(); + log(i("The subcommands that take [nodeId] as an argument ('" + LIST.text() + "', '" + CONTENTION.text() + "' and '" + VALIDATE_INDEXES.text() + "') will be executed on the given node or on all server nodes if the option is not specified. Other commands will run on a random server node.")); + nl(); + nl(); + log(i("Subcommands:")); + + usageCache(LIST, "regexPattern", "[groups|seq]", "[nodeId]", op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE.text())); + usageCache(CONTENTION, "minQueueSize", "[nodeId]", "[maxPrint]"); + usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]"); + usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", "[nodeId]", op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K"))); + usageCache(DISTRIBUTION, or("nodeId", NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attName1,...,attrNameN")); + usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN"); nl(); } @@ -740,8 +830,11 @@ private void cacheView(GridClient client, CacheArguments cacheArgs) throws GridC VisorViewCacheTaskResult res = executeTaskByNameOnNode( client, VisorViewCacheTask.class.getName(), taskArg, cacheArgs.nodeId()); - for (CacheInfo info : res.cacheInfos()) - info.print(cacheArgs.cacheCommand()); + if (cacheArgs.fullConfig() && cacheArgs.cacheCommand() == CACHES) + cachesConfig(client, cacheArgs, res); + else + printCacheInfos(res.cacheInfos(), cacheArgs.cacheCommand()); + } /** @@ -816,6 +909,101 @@ private void cacheDistribution(GridClient client, CacheArguments cacheArgs) thro res.print(System.out); } + /** + * @param client Client. + * @param cacheArgs Cache args. + * @param viewRes Cache view task result. + */ + private void cachesConfig(GridClient client, CacheArguments cacheArgs, + VisorViewCacheTaskResult viewRes) throws GridClientException { + VisorCacheConfigurationCollectorTaskArg taskArg = new VisorCacheConfigurationCollectorTaskArg(cacheArgs.regex()); + + UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId(); + + Map res = + executeTaskByNameOnNode(client, VisorCacheConfigurationCollectorTask.class.getName(), taskArg, nodeId); + + Map cacheToMapped = + viewRes.cacheInfos().stream().collect(Collectors.toMap(x -> x.getCacheName(), x -> x.getMapped())); + + printCachesConfig(res, cacheArgs.outputFormat(), cacheToMapped); + + } + + /** + * Prints caches info. + * + * @param infos Caches info. + * @param cmd Command. + */ + private void printCacheInfos(Collection infos, VisorViewCacheCmd cmd) { + for (CacheInfo info : infos) { + Map map = info.toMap(cmd); + + SB sb = new SB("["); + + for (Map.Entry e : map.entrySet()) + sb.a(e.getKey()).a("=").a(e.getValue()).a(", "); + + sb.setLength(sb.length() - 2); + + sb.a("]"); + + log(sb.toString()); + } + } + + /** + * Prints caches config. + * + * @param caches Caches config. + * @param outputFormat Output format. + * @param cacheToMapped Map cache name to mapped. + */ + private void printCachesConfig( + Map caches, + OutputFormat outputFormat, + Map cacheToMapped + ) { + + for (Map.Entry entry : caches.entrySet()) { + String cacheName = entry.getKey(); + + switch (outputFormat) { + case MULTI_LINE: + Map params = mapToPairs(entry.getValue()); + + params.put("Mapped", cacheToMapped.get(cacheName)); + + log("[cache = '%s']%n", cacheName); + + for (Map.Entry innerEntry : params.entrySet()) + log("%s: %s%n", innerEntry.getKey(), innerEntry.getValue()); + + nl(); + + break; + + default: + int mapped = cacheToMapped.get(cacheName); + + log("%s: %s %s=%s%n", entry.getKey(), toString(entry.getValue()), "mapped", mapped); + + break; + } + } + } + + /** + * Invokes toString() method and cuts class name from result string. + * + * @param cfg Visor cache configuration for invocation. + * @return String representation without class name in begin of string. + */ + private String toString(VisorCacheConfiguration cfg) { + return cfg.toString().substring(cfg.getClass().getSimpleName().length() + 1); + } + /** * @param client Client. * @param cacheArgs Cache args. @@ -954,8 +1142,9 @@ private void baselinePrint0(VisorBaselineTaskResult res) { log("Baseline nodes:"); for (VisorBaselineNode node : baseline.values()) { - log(" ConsistentID=" + node.getConsistentId() + ", STATE=" + - (srvs.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE")); + boolean online = srvs.containsKey(node.getConsistentId()); + + log(i("ConsistentID=" + node.getConsistentId() + ", STATE=" + (online ? "ONLINE" : "OFFLINE"), 2)); } log(DELIM); @@ -976,7 +1165,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) { log("Other nodes:"); for (VisorBaselineNode node : others) - log(" ConsistentID=" + node.getConsistentId()); + log(i("ConsistentID=" + node.getConsistentId(), 2)); log("Number of other nodes: " + others.size()); } @@ -1209,10 +1398,10 @@ private void printUnusedWalSegments0(VisorWalTaskResult taskRes) { VisorClusterNode node = nodesInfo.get(entry.getKey()); log("Node=" + node.getConsistentId()); - log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())); + log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()), 2)); for (String fileName : entry.getValue()) - log(" " + fileName); + log(i(fileName)); nl(); } @@ -1221,8 +1410,8 @@ private void printUnusedWalSegments0(VisorWalTaskResult taskRes) { VisorClusterNode node = nodesInfo.get(entry.getKey()); log("Node=" + node.getConsistentId()); - log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())); - log(" failed with error: " + entry.getValue().getMessage()); + log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())), 2); + log(i("failed with error: " + entry.getValue().getMessage())); nl(); } } @@ -1244,7 +1433,7 @@ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) { VisorClusterNode node = nodesInfo.get(entry.getKey()); log("Node=" + node.getConsistentId()); - log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())); + log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())), 2); nl(); } @@ -1252,8 +1441,8 @@ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) { VisorClusterNode node = nodesInfo.get(entry.getKey()); log("Node=" + node.getConsistentId()); - log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())); - log(" failed with error: " + entry.getValue().getMessage()); + log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())), 2); + log(i("failed with error: " + entry.getValue().getMessage())); nl(); } } @@ -1286,11 +1475,170 @@ private boolean isConnectionError(Throwable e) { */ private void usage(String desc, Command cmd, String... args) { log(desc); - log(" control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " + - " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args)); + log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + cmd.text() + " " + String.join(" ", args), 2)); nl(); } + /** + * Print cache command usage with default indention. + * + * @param cmd Cache command. + * @param args Cache command arguments. + */ + private void usageCache(CacheCommand cmd, String... args) { + usageCache(1, cmd, args); + } + + /** + * Print cache command usage. + * + * @param indentsNum Number of indents. + * @param cmd Cache command. + * @param args Cache command arguments. + */ + private void usageCache(int indentsNum, CacheCommand cmd, String... args) { + log(i(DELIM, indentsNum)); + nl(); + log(i(CACHE.text() + " " + cmd.text() + " " + String.join(" ", args), indentsNum++)); + nl(); + log(i(getCacheSubcommandDesc(cmd), indentsNum)); + nl(); + + Map paramsDesc = createCacheArgsDesc(cmd); + + if (!paramsDesc.isEmpty()) { + log(i("Parameters:", indentsNum)); + + usageCacheParams(paramsDesc, indentsNum + 1); + + nl(); + } + } + + /** + * Print cache command arguments usage. + * + * @param paramsDesc Cache command arguments description. + * @param indentsNum Number of indents. + */ + private void usageCacheParams(Map paramsDesc, int indentsNum) { + int maxParamLen = paramsDesc.keySet().stream().max(Comparator.comparingInt(String::length)).get().length(); + + for (Map.Entry param : paramsDesc.entrySet()) + log(i(extendToLen(param.getKey(), maxParamLen) + INDENT + "- " + param.getValue(), indentsNum)); + } + + /** + * Appends spaces to end of input string for extending to needed length. + * + * @param s Input string. + * @param targetLen Needed length. + * @return String with appended spaces on the end. + */ + private String extendToLen(String s, int targetLen) { + assert targetLen >= 0; + assert s.length() <= targetLen; + + if (s.length() == targetLen) + return s; + + SB sb = new SB(targetLen); + + sb.a(s); + + for (int i = 0; i < targetLen - s.length(); i++) + sb.a(" "); + + return sb.toString(); + } + + /** + * Gets cache command description by cache command. + * + * @param cmd Cache command. + * @return Cache command description. + */ + private String getCacheSubcommandDesc(CacheCommand cmd) { + switch (cmd) { + case LIST: + return "Show information about caches, groups or sequences that match a regular expression. When executed without parameters, this subcommand prints the list of caches."; + + case CONTENTION: + return "Show the keys that are point of contention for multiple transactions."; + + case IDLE_VERIFY: + return "Verify counters and hash sums of primary and backup partitions for the specified caches on an idle cluster and print out the differences, if any."; + + case VALIDATE_INDEXES: + return "Validate indexes on an idle cluster and print out the keys that are missing in the indexes."; + + case DISTRIBUTION: + return "Prints the information about partition distribution."; + + case RESET_LOST_PARTITIONS: + return "Reset the state of lost partitions for the specified caches."; + + default: + throw new IllegalArgumentException("Unknown command: " + cmd); + } + } + + /** + * Gets cache command arguments description by cache command. + * + * @param cmd Cache command. + * @return Cache command arguments description. + */ + private Map createCacheArgsDesc(CacheCommand cmd) { + Map map = U.newLinkedHashMap(16); + switch (cmd) { + case LIST: + map.put(CONFIG, "print a all configuration parameters for each cache."); + map.put(OUTPUT_FORMAT + " " + MULTI_LINE.text(), "print configuration parameters per line. This option has effect only when used with " + CONFIG + " and without [groups|seq]."); + + break; + case VALIDATE_INDEXES: + map.put(VI_CHECK_FIRST + " N", "validate only the first N keys"); + map.put(VI_CHECK_THROUGH + " K", "validate every Kth key"); + + break; + } + return map; + } + + /** + * Join input parameters with space and wrap optional braces {@code []}. + * + * @param param First input parameter. + * @param params Other input parameter. + * @return Joined parameters wrapped optional braces. + */ + private static String op(String param, String... params) { + if (params == null || params.length == 0) + return "[" + param + "]"; + + return "[" + param + " " + String.join(" ", params) + "]"; + } + + /** + * Concatenates input parameters to single string with OR delimiter {@code |}. + * + * @param param1 First parameter. + * @param params Remaining parameters. + * @return Concatenated string. + */ + private static String or(String param1, String... params) { + if (params.length == 0) + return param1; + + SB sb = new SB(param1); + + for (String param : params) + sb.a("|").a(param); + + return sb.toString(); + } + /** * Extract next argument. * @@ -1612,27 +1960,27 @@ else if (CMD_SKIP_ZEROS.equals(nextArg)) case DISTRIBUTION: String nodeIdStr = nextArg("Node id expected or null"); - if (!"null".equals(nodeIdStr)) + if (!NULL.equals(nodeIdStr)) cacheArgs.nodeId(UUID.fromString(nodeIdStr)); while (hasNextCacheArg()) { String nextArg = nextArg(""); - if (CMD_USER_ATTRIBUTES.equals(nextArg)){ + if (CMD_USER_ATTRIBUTES.equals(nextArg)) { nextArg = nextArg("User attributes are expected to be separated by commas"); - Set userAttributes = new HashSet(); + Set userAttrs = new HashSet<>(); - for (String userAttribute:nextArg.split(",")) - userAttributes.add(userAttribute.trim()); + for (String userAttribute : nextArg.split(",")) + userAttrs.add(userAttribute.trim()); - cacheArgs.setUserAttributes(userAttributes); + cacheArgs.setUserAttributes(userAttrs); nextArg = (hasNextCacheArg()) ? nextArg("") : null; } - if (nextArg!=null) + if (nextArg != null) parseCacheNames(nextArg, cacheArgs); } @@ -1643,20 +1991,36 @@ else if (CMD_SKIP_ZEROS.equals(nextArg)) break; - default: + case LIST: cacheArgs.regex(nextArg("Regex is expected")); - if (hasNextCacheArg()) { - String tmp = nextArg(""); + VisorViewCacheCmd cacheCmd = CACHES; + + OutputFormat outputFormat = SINGLE_LINE; + + while (hasNextCacheArg()) { + String tmp = nextArg("").toLowerCase(); switch (tmp) { case "groups": - cacheArgs.cacheCommand(GROUPS); + cacheCmd = GROUPS; break; case "seq": - cacheArgs.cacheCommand(SEQ); + cacheCmd = SEQ; + + break; + + case OUTPUT_FORMAT: + String tmp2 = nextArg("output format must be defined!").toLowerCase(); + + outputFormat = OutputFormat.fromConsoleName(tmp2); + + break; + + case CONFIG: + cacheArgs.fullConfig(true); break; @@ -1665,7 +2029,13 @@ else if (CMD_SKIP_ZEROS.equals(nextArg)) } } + cacheArgs.cacheCommand(cacheCmd); + cacheArgs.outputFormat(outputFormat); + break; + + default: + throw new IllegalArgumentException("Unknown --cache subcommand " + cmd); } if (hasNextCacheArg()) @@ -1860,6 +2230,107 @@ private boolean isCommandOrOption(String raw) { return raw != null && raw.contains("--"); } + /** + * Maps VisorCacheConfiguration to key-value pairs. + * + * @param cfg Visor cache configuration. + * @return map of key-value pairs. + */ + private Map mapToPairs(VisorCacheConfiguration cfg) { + Map params = new LinkedHashMap<>(); + + VisorCacheAffinityConfiguration affinityCfg = cfg.getAffinityConfiguration(); + VisorCacheNearConfiguration nearCfg = cfg.getNearConfiguration(); + VisorCacheRebalanceConfiguration rebalanceCfg = cfg.getRebalanceConfiguration(); + VisorCacheEvictionConfiguration evictCfg = cfg.getEvictionConfiguration(); + VisorCacheStoreConfiguration storeCfg = cfg.getStoreConfiguration(); + VisorQueryConfiguration qryCfg = cfg.getQueryConfiguration(); + + params.put("Name", cfg.getName()); + params.put("Group", cfg.getGroupName()); + params.put("Dynamic Deployment ID", cfg.getDynamicDeploymentId()); + params.put("System", cfg.isSystem()); + + params.put("Mode", cfg.getMode()); + params.put("Atomicity Mode", cfg.getAtomicityMode()); + params.put("Statistic Enabled", cfg.isStatisticsEnabled()); + params.put("Management Enabled", cfg.isManagementEnabled()); + + params.put("On-heap cache enabled", cfg.isOnheapCacheEnabled()); + params.put("Partition Loss Policy", cfg.getPartitionLossPolicy()); + params.put("Query Parallelism", cfg.getQueryParallelism()); + params.put("Copy On Read", cfg.isCopyOnRead()); + params.put("Listener Configurations", cfg.getListenerConfigurations()); + params.put("Load Previous Value", cfg.isLoadPreviousValue()); + params.put("Memory Policy Name", cfg.getMemoryPolicyName()); + params.put("Node Filter", cfg.getNodeFilter()); + params.put("Read From Backup", cfg.isReadFromBackup()); + params.put("Topology Validator", cfg.getTopologyValidator()); + + params.put("Time To Live Eager Flag", cfg.isEagerTtl()); + + params.put("Write Synchronization Mode", cfg.getWriteSynchronizationMode()); + params.put("Invalidate", cfg.isInvalidate()); + + params.put("Affinity Function", affinityCfg.getFunction()); + params.put("Affinity Backups", affinityCfg.getPartitionedBackups()); + params.put("Affinity Partitions", affinityCfg.getPartitions()); + params.put("Affinity Exclude Neighbors", affinityCfg.isExcludeNeighbors()); + params.put("Affinity Mapper", affinityCfg.getMapper()); + + params.put("Rebalance Mode", rebalanceCfg.getMode()); + params.put("Rebalance Batch Size", rebalanceCfg.getBatchSize()); + params.put("Rebalance Timeout", rebalanceCfg.getTimeout()); + params.put("Rebalance Delay", rebalanceCfg.getPartitionedDelay()); + params.put("Time Between Rebalance Messages", rebalanceCfg.getThrottle()); + params.put("Rebalance Batches Count", rebalanceCfg.getBatchesPrefetchCnt()); + params.put("Rebalance Cache Order", rebalanceCfg.getRebalanceOrder()); + + params.put("Eviction Policy Enabled", (evictCfg.getPolicy() != null)); + params.put("Eviction Policy Factory", evictCfg.getPolicy()); + params.put("Eviction Policy Max Size", evictCfg.getPolicyMaxSize()); + params.put("Eviction Filter", evictCfg.getFilter()); + + params.put("Near Cache Enabled", nearCfg.isNearEnabled()); + params.put("Near Start Size", nearCfg.getNearStartSize()); + params.put("Near Eviction Policy Factory", nearCfg.getNearEvictPolicy()); + params.put("Near Eviction Policy Max Size", nearCfg.getNearEvictMaxSize()); + + params.put("Default Lock Timeout", cfg.getDefaultLockTimeout()); + params.put("Query Entities", cfg.getQueryEntities()); + params.put("Cache Interceptor", cfg.getInterceptor()); + + params.put("Store Enabled", storeCfg.isEnabled()); + params.put("Store Class", storeCfg.getStore()); + params.put("Store Factory Class", storeCfg.getStoreFactory()); + params.put("Store Keep Binary", storeCfg.isStoreKeepBinary()); + params.put("Store Read Through", storeCfg.isReadThrough()); + params.put("Store Write Through", storeCfg.isWriteThrough()); + params.put("Store Write Coalescing", storeCfg.getWriteBehindCoalescing()); + + params.put("Write-Behind Enabled", storeCfg.isWriteBehindEnabled()); + params.put("Write-Behind Flush Size", storeCfg.getFlushSize()); + params.put("Write-Behind Frequency", storeCfg.getFlushFrequency()); + params.put("Write-Behind Flush Threads Count", storeCfg.getFlushThreadCount()); + params.put("Write-Behind Batch Size", storeCfg.getBatchSize()); + + params.put("Concurrent Asynchronous Operations Number", cfg.getMaxConcurrentAsyncOperations()); + + params.put("Loader Factory Class Name", cfg.getLoaderFactory()); + params.put("Writer Factory Class Name", cfg.getWriterFactory()); + params.put("Expiry Policy Factory Class Name", cfg.getExpiryPolicyFactory()); + + params.put("Query Execution Time Threshold", qryCfg.getLongQueryWarningTimeout()); + params.put("Query Escaped Names", qryCfg.isSqlEscapeAll()); + params.put("Query SQL Schema", qryCfg.getSqlSchema()); + params.put("Query SQL functions", qryCfg.getSqlFunctionClasses()); + params.put("Query Indexed Types", qryCfg.getIndexedTypes()); + params.put("Maximum payload size for offheap indexes", cfg.getSqlIndexMaxInlineSize()); + params.put("Query Metrics History Size", cfg.getQueryDetailMetricsSize()); + + return params; + } + /** * Parse and execute command. * @@ -1876,27 +2347,23 @@ public int execute(List rawArgs) { if (F.isEmpty(rawArgs) || (rawArgs.size() == 1 && CMD_HELP.equalsIgnoreCase(rawArgs.get(0)))) { log("This utility can do the following commands:"); - usage(" Activate cluster:", ACTIVATE); - usage(" Deactivate cluster:", DEACTIVATE, " [" + CMD_AUTO_CONFIRMATION + "]"); - usage(" Print current cluster state:", STATE); - usage(" Print cluster baseline topology:", BASELINE); - usage(" Add nodes into baseline topology:", BASELINE, " add consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]"); - usage(" Remove nodes from baseline topology:", BASELINE, " remove consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]"); - usage(" Set baseline topology:", BASELINE, " set consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]"); - usage(" Set baseline topology based on version:", BASELINE, " version topologyVersion [" + CMD_AUTO_CONFIRMATION + "]"); - usage(" List or kill transactions:", TX, " [xid XID] [minDuration SECONDS] " + - "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " + - "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [" + CMD_AUTO_CONFIRMATION + "]"); + usage(i("Activate cluster:"), ACTIVATE); + usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION)); + usage(i("Print current cluster state:"), STATE); + usage(i("Print cluster baseline topology:"), BASELINE); + usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION)); + usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION)); if (enableExperimental) { - usage(" Print absolute paths of unused archived wal segments on each node:", WAL, - " print [consistentId1,consistentId2,....,consistentIdN]"); - usage(" Delete unused archived wal segments on each node:", WAL, - " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]"); + usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]"); + usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION)); } - log(" View caches information in a cluster. For more details type:"); - log(" control.sh --cache help"); + log(i("View caches information in a cluster. For more details type:")); + log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2)); nl(); log("By default commands affecting the cluster require interactive confirmation."); @@ -1904,24 +2371,30 @@ public int execute(List rawArgs) { nl(); log("Default values:"); - log(" HOST_OR_IP=" + DFLT_HOST); - log(" PORT=" + DFLT_PORT); - log(" PING_INTERVAL=" + DFLT_PING_INTERVAL); - log(" PING_TIMEOUT=" + DFLT_PING_TIMEOUT); + log(i("HOST_OR_IP=" + DFLT_HOST, 2)); + log(i("PORT=" + DFLT_PORT, 2)); + log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2)); + log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2)); nl(); log("Exit codes:"); - log(" " + EXIT_CODE_OK + " - successful execution."); - log(" " + EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments."); - log(" " + EXIT_CODE_CONNECTION_FAILED + " - connection failed."); - log(" " + ERR_AUTHENTICATION_FAILED + " - authentication failed."); - log(" " + EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error."); + log(i(EXIT_CODE_OK + " - successful execution.", 2)); + log(i(EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.", 2)); + log(i(EXIT_CODE_CONNECTION_FAILED + " - connection failed.", 2)); + log(i(ERR_AUTHENTICATION_FAILED + " - authentication failed.", 2)); + log(i(EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.", 2)); return EXIT_CODE_OK; } Arguments args = parseAndValidate(rawArgs); + if (args.command() == CACHE && args.cacheArgs().command() == HELP) { + printCacheHelp(); + + return EXIT_CODE_OK; + } + if (!args.autoConfirmation() && !confirm(args)) { log("Operation cancelled."); @@ -1980,7 +2453,7 @@ public int execute(List rawArgs) { } } - return 0; + return EXIT_CODE_OK; } catch (IllegalArgumentException e) { return error(EXIT_CODE_INVALID_ARGUMENTS, "Check arguments.", e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java new file mode 100644 index 0000000000000..356cb4b2e88e5 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.commandline; + +import org.jetbrains.annotations.NotNull; + +/** + * + */ +public enum OutputFormat { + /** Single line. */ + SINGLE_LINE("single-line"), + + /** Multi line. */ + MULTI_LINE("multi-line"); + + /** */ + private final String text; + + /** */ + OutputFormat(String text) { + this.text = text; + } + + /** + * @return Text. + */ + public String text() { + return text; + } + + /** + * Converts format name in console to enumerated value. + * + * @param text Format name in console. + * @return Enumerated value. + * @throws IllegalArgumentException If enumerated value not found. + */ + public static OutputFormat fromConsoleName(@NotNull String text) { + for (OutputFormat format : values()) { + if (format.text.equals(text)) + return format; + } + + throw new IllegalArgumentException("Unknown output format " + text); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return text; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java index 97d234aeb879d..f7146103a086d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java @@ -18,6 +18,7 @@ import java.util.Set; import java.util.UUID; +import org.apache.ignite.internal.commandline.OutputFormat; import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd; import org.jetbrains.annotations.Nullable; @@ -64,6 +65,22 @@ public class CacheArguments { /** Additional user attributes in result. Set of attribute names whose values will be searched in ClusterNode.attributes(). */ private Set userAttributes; + /** Output format. */ + private OutputFormat outputFormat; + + /** Full config flag. */ + private boolean fullConfig; + + /** + * @return Full config flag. + */ + public boolean fullConfig(){ return fullConfig; } + + /** + * @param fullConfig New full config flag. + */ + public void fullConfig(boolean fullConfig) { this.fullConfig = fullConfig; } + /** * @return Command. */ @@ -245,4 +262,14 @@ public Set getUserAttributes() { public void setUserAttributes(Set userAttrs) { userAttributes = userAttrs; } + + /** + * @return Output format. + */ + public OutputFormat outputFormat() { return outputFormat; } + + /** + * @param outputFormat New output format. + */ + public void outputFormat(OutputFormat outputFormat) { this.outputFormat = outputFormat; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java index 31c0b3fcbe2d0..01231990050bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java @@ -20,7 +20,8 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; - +import java.util.LinkedHashMap; +import java.util.Map; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; @@ -238,7 +239,7 @@ public CacheAtomicityMode getAtomicityMode() { } /** - * @param atomicityMode + * @param atomicityMode Atomicity mode. */ public void setAtomicityMode(CacheAtomicityMode atomicityMode) { this.atomicityMode = atomicityMode; @@ -272,32 +273,74 @@ public void setAffinityClsName(String affinityClsName) { this.affinityClsName = affinityClsName; } + /** + * Gets name of info for multi line output depending on cache command. + * + * @param cmd Cache command. + * @return Header. + */ + public Object name(VisorViewCacheCmd cmd) { + switch (cmd) { + case CACHES: + return getCacheName(); + + case GROUPS: + return getGrpName(); + + case SEQ: + return getSeqName(); + + default: + throw new IllegalArgumentException("Unknown cache subcommand " + cmd); + } + } + /** * @param cmd Command. */ - public void print(VisorViewCacheCmd cmd) { - if (cmd == null) - cmd = VisorViewCacheCmd.CACHES; + public Map toMap(VisorViewCacheCmd cmd) { + Map map; switch (cmd) { case SEQ: - System.out.println("[seqName=" + getSeqName() + ", curVal=" + seqVal + ']'); + map = new LinkedHashMap<>(2); + + map.put("seqName", getSeqName()); + map.put("curVal", seqVal); break; case GROUPS: - System.out.println("[grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", cachesCnt=" + getCachesCnt() + - ", prim=" + getPartitions() + ", mapped=" + getMapped() + ", mode=" + getMode() + - ", atomicity=" + getAtomicityMode() + ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']'); + map = new LinkedHashMap<>(10); + + map.put("grpName", getGrpName()); + map.put("grpId", getGrpId()); + map.put("cachesCnt", getCachesCnt()); + map.put("prim", getPartitions()); + map.put("mapped", getMapped()); + map.put("mode", getMode()); + map.put("atomicity", getAtomicityMode()); + map.put("backups", getBackupsCnt()); + map.put("affCls", getAffinityClsName()); break; default: - System.out.println("[cacheName=" + getCacheName() + ", cacheId=" + getCacheId() + - ", grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", prim=" + getPartitions() + - ", mapped=" + getMapped() + ", mode=" + getMode() + ", atomicity=" + getAtomicityMode() + - ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']'); + map = new LinkedHashMap<>(10); + + map.put("cacheName", getCacheName()); + map.put("cacheId", getCacheId()); + map.put("grpName", getGrpName()); + map.put("grpId", getGrpId()); + map.put("prim", getPartitions()); + map.put("mapped", getMapped()); + map.put("mode", getMode()); + map.put("atomicity", getAtomicityMode()); + map.put("backups", getBackupsCnt()); + map.put("affCls", getAffinityClsName()); } + + return map; } /** {@inheritDoc} */ @@ -345,4 +388,4 @@ public void print(VisorViewCacheCmd cmd) { @Override public String toString() { return S.toString(CacheInfo.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java index b0126788db819..1e9faaabaded2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java @@ -36,6 +36,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; + import static org.apache.ignite.internal.visor.util.VisorTaskUtils.compactClass; import static org.apache.ignite.internal.visor.util.VisorTaskUtils.compactIterable; @@ -489,8 +490,8 @@ public int getQueryDetailMetricsSize() { } /** - * @return {@code true} if data can be read from backup node or {@code false} if data always - * should be read from primary node and never from backup. + * @return {@code true} if data can be read from backup node or {@code false} if data always should be read from + * primary node and never from backup. */ public boolean isReadFromBackup() { return readFromBackup; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java index f72cd7de09421..9cf2c89b8855d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java @@ -19,6 +19,7 @@ import java.util.Collection; import java.util.Map; +import java.util.regex.Pattern; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.util.typedef.F; @@ -28,7 +29,7 @@ import org.apache.ignite.lang.IgniteUuid; /** - * Job that collect cache metrics from node. + * Job that collect cache configuration from node. */ public class VisorCacheConfigurationCollectorJob extends VisorJob> { @@ -38,7 +39,7 @@ public class VisorCacheConfigurationCollectorJob /** * Create job with given argument. * - * @param arg Whether to collect metrics for all caches or for specified cache name only. + * @param arg Whether to collect metrics for all caches or for specified cache name only or by regex. * @param debug Debug flag. */ public VisorCacheConfigurationCollectorJob(VisorCacheConfigurationCollectorTaskArg arg, boolean debug) { @@ -49,18 +50,24 @@ public VisorCacheConfigurationCollectorJob(VisorCacheConfigurationCollectorTaskA @Override protected Map run(VisorCacheConfigurationCollectorTaskArg arg) { Collection> caches = ignite.context().cache().jcaches(); - Collection cacheNames = arg.getCacheNames(); + Pattern ptrn = arg.getRegex() != null ? Pattern.compile(arg.getRegex()) : null; - boolean all = F.isEmpty(cacheNames); + boolean all = F.isEmpty(arg.getCacheNames()); - Map res = U.newHashMap(all ? caches.size() : cacheNames.size()); + boolean hasPtrn = ptrn != null; + + Map res = U.newHashMap(caches.size()); for (IgniteCacheProxy cache : caches) { String cacheName = cache.getName(); - if (all || cacheNames.contains(cacheName)) { - res.put(cacheName, config(cache.getConfiguration(CacheConfiguration.class), - cache.context().dynamicDeploymentId())); + boolean matched = hasPtrn ? ptrn.matcher(cacheName).find() : all || arg.getCacheNames().contains(cacheName); + + if (matched) { + VisorCacheConfiguration cfg = + config(cache.getConfiguration(CacheConfiguration.class), cache.context().dynamicDeploymentId()); + + res.put(cacheName, cfg); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java index a0b43dbcf890d..a91f0ca4e2e70 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java @@ -21,6 +21,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.Collection; +import java.util.regex.Pattern; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; @@ -32,9 +33,12 @@ public class VisorCacheConfigurationCollectorTaskArg extends VisorDataTransferOb /** */ private static final long serialVersionUID = 0L; - /** Collection of cache deployment IDs. */ + /** Collection of cache names. */ private Collection cacheNames; + /** Cache name regexp. */ + private String regex; + /** * Default constructor. */ @@ -49,6 +53,16 @@ public VisorCacheConfigurationCollectorTaskArg(Collection cacheNames) { this.cacheNames = cacheNames; } + /** + * @param regex Cache name regexp. + */ + public VisorCacheConfigurationCollectorTaskArg(String regex) { + // Checks, that regex is correct. + Pattern.compile(regex); + + this.regex = regex; + } + /** * @return Collection of cache deployment IDs. */ @@ -56,14 +70,30 @@ public Collection getCacheNames() { return cacheNames; } + /** + * @return Cache name regexp. + */ + public String getRegex() { + return regex; + } + + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V2; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeCollection(out, cacheNames); + U.writeString(out, regex); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { cacheNames = U.readCollection(in); + + if (getProtocolVersion() > V1) + regex = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java index 5fcd66d18d082..6bc2369406ee9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java @@ -1,19 +1,19 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.ignite.internal.visor.verify; @@ -23,7 +23,6 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; -import org.jetbrains.annotations.Nullable; /** * @@ -36,13 +35,13 @@ public class VisorViewCacheTaskArg extends VisorDataTransferObject { private String regex; /** Type. */ - private @Nullable VisorViewCacheCmd cmd; + private VisorViewCacheCmd cmd; /** * @param regex Regex. * @param cmd Command. */ - public VisorViewCacheTaskArg(String regex, @Nullable VisorViewCacheCmd cmd) { + public VisorViewCacheTaskArg(String regex, VisorViewCacheCmd cmd) { this.regex = regex; this.cmd = cmd; } diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index cf2bd2873e72a..6cbbb3fb2cbce 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -17,9 +17,6 @@ package org.apache.ignite.util; -import javax.cache.processor.EntryProcessor; -import javax.cache.processor.EntryProcessorException; -import javax.cache.processor.MutableEntry; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; @@ -42,6 +39,9 @@ import java.util.concurrent.atomic.LongAdder; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.cache.processor.EntryProcessor; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.MutableEntry; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteAtomicSequence; import org.apache.ignite.IgniteCache; @@ -58,6 +58,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.commandline.CommandHandler; +import org.apache.ignite.internal.commandline.OutputFormat; import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; @@ -81,6 +82,7 @@ import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.tx.VisorTxInfo; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; @@ -102,6 +104,8 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR; +import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE; +import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE; import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREMIX; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; @@ -231,6 +235,10 @@ protected int execute(ArrayList args) { // Add force to avoid interactive confirmation args.add(CMD_AUTO_CONFIRMATION); + SB sb = new SB(); + + args.forEach(arg -> sb.a(arg).a(" ")); + return new CommandHandler().execute(args); } @@ -1254,6 +1262,60 @@ public void testCacheAffinity() throws Exception { assertTrue(testOut.toString().contains("affCls=RendezvousAffinityFunction")); } + /** */ + public void testCacheConfigNoOutputFormat() throws Exception { + testCacheConfig(null); + } + + /** */ + public void testCacheConfigSingleLineOutputFormat() throws Exception { + testCacheConfig(SINGLE_LINE); + } + + /** */ + public void testCacheConfigMultiLineOutputFormat() throws Exception { + testCacheConfig(MULTI_LINE); + } + + /** */ + private void testCacheConfig(OutputFormat outputFormat) throws Exception { + Ignite ignite = startGrid(); + + ignite.cluster().active(true); + + IgniteCache cache1 = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); + + for (int i = 0; i < 100; i++) + cache1.put(i, i); + + injectTestSystemOut(); + + int exitCode; + + if (outputFormat == null) + exitCode = execute("--cache", "list", ".*", "--config"); + else + exitCode = execute("--cache", "list", ".*", "--config", "--output-format", outputFormat.text()); + + assertEquals(EXIT_CODE_OK, exitCode); + + String outStr = testOut.toString(); + + if (outputFormat == null || outputFormat == SINGLE_LINE) { + assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME)); + assertTrue(outStr.contains("partitions=32")); + assertTrue(outStr.contains("function=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction")); + } + else if (outputFormat == MULTI_LINE) { + assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + "']")); + assertTrue(outStr.contains("Affinity Partitions: 32")); + assertTrue(outStr.contains("Affinity Function: o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction")); + } + } + /** * */ @@ -1294,7 +1356,7 @@ public void testCacheDistribution() throws Exception { assertTrue(lastRowIndex > 0); // Last row is empty, but the previous line contains data - lastRowIndex = log.lastIndexOf('\n', lastRowIndex-1); + lastRowIndex = log.lastIndexOf('\n', lastRowIndex - 1); assertTrue(lastRowIndex > 0); From 5214bac64438449f7a2d229c6bb6a50fc423f9f2 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Thu, 25 Oct 2018 18:10:13 +0700 Subject: [PATCH 043/403] IGNITE-9853 control.sh: Added output of cache configurations. Fixes #4956. (cherry picked from commit a18f71ce5c89465e7b32fca71755208ac8976128) --- .../visor/cache/VisorCacheConfigurationCollectorTaskArg.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java index a91f0ca4e2e70..b4b8143be0de0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java @@ -92,7 +92,7 @@ public String getRegex() { @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { cacheNames = U.readCollection(in); - if (getProtocolVersion() > V1) + if (protoVer > V1) regex = U.readString(in); } From 0f556fe62149a5897609bc674fdbf1fbb58c4f0d Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Fri, 26 Oct 2018 11:31:51 +0300 Subject: [PATCH 044/403] IGNITE-9454: Fixed SecurityPermissionSetBuilder. This closes #5066. --- .../SecurityPermissionSetBuilder.java | 9 +++++-- .../SecurityPermissionSetBuilderTest.java | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java index abac541ffdb68..659613aa0e8c5 100644 --- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java +++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilder.java @@ -126,6 +126,11 @@ public SecurityPermissionSetBuilder appendServicePermissions(String name, Securi * @return {@link SecurityPermissionSetBuilder} refer to same permission builder. */ public SecurityPermissionSetBuilder appendCachePermissions(String name, SecurityPermission... perms) { + for (SecurityPermission perm : perms) { + if (perm == SecurityPermission.CACHE_CREATE || perm == SecurityPermission.CACHE_DESTROY) + throw new IgniteException(perm + " should be assigned as system permission, not cache permission"); + } + validate(toCollection("CACHE_"), perms); append(cachePerms, name, toCollection(perms)); @@ -140,7 +145,7 @@ public SecurityPermissionSetBuilder appendCachePermissions(String name, Security * @return {@link SecurityPermissionSetBuilder} refer to same permission builder. */ public SecurityPermissionSetBuilder appendSystemPermissions(SecurityPermission... perms) { - validate(toCollection("EVENTS_", "ADMIN_"), perms); + validate(toCollection("EVENTS_", "ADMIN_", "CACHE_CREATE", "CACHE_DESTROY", "JOIN_AS_SERVER"), perms); sysPerms.addAll(toCollection(perms)); @@ -194,7 +199,7 @@ private void validate(Collection ptrns, SecurityPermission perm) { private final Collection toCollection(T... perms) { assert perms != null; - Collection col = U.newHashSet(perms.length); + Collection col = U.newLinkedHashSet(perms.length); Collections.addAll(col, perms); diff --git a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java index 0ac7bc739f179..338034f6d3264 100644 --- a/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/plugin/security/SecurityPermissionSetBuilderTest.java @@ -27,9 +27,12 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_VIEW; +import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_CREATE; +import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_DESTROY; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_PUT; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_REMOVE; +import static org.apache.ignite.plugin.security.SecurityPermission.JOIN_AS_SERVER; import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_DEPLOY; import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_INVOKE; import static org.apache.ignite.plugin.security.SecurityPermission.EVENTS_ENABLE; @@ -65,7 +68,7 @@ public void testPermissionBuilder() { exp.setServicePermissions(permSrvc); - exp.setSystemPermissions(permissions(ADMIN_VIEW, EVENTS_ENABLE)); + exp.setSystemPermissions(permissions(ADMIN_VIEW, EVENTS_ENABLE, JOIN_AS_SERVER, CACHE_CREATE, CACHE_DESTROY)); final SecurityPermissionSetBuilder permsBuilder = new SecurityPermissionSetBuilder(); @@ -80,7 +83,7 @@ public void testPermissionBuilder() { assertThrows(log, new Callable() { @Override public Object call() throws Exception { - permsBuilder.appendTaskPermissions("task", CACHE_READ); + permsBuilder.appendTaskPermissions("task", CACHE_READ, JOIN_AS_SERVER); return null; } }, IgniteException.class, @@ -93,7 +96,7 @@ public void testPermissionBuilder() { return null; } }, IgniteException.class, - "you can assign permission only start with [EVENTS_, ADMIN_], but you try TASK_EXECUTE" + "you can assign permission only start with [EVENTS_, ADMIN_, CACHE_CREATE, CACHE_DESTROY, JOIN_AS_SERVER], but you try TASK_EXECUTE" ); assertThrows(log, new Callable() { @@ -102,7 +105,16 @@ public void testPermissionBuilder() { return null; } }, IgniteException.class, - "you can assign permission only start with [EVENTS_, ADMIN_], but you try SERVICE_INVOKE" + "you can assign permission only start with [EVENTS_, ADMIN_, CACHE_CREATE, CACHE_DESTROY, JOIN_AS_SERVER], but you try SERVICE_INVOKE" + ); + + assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + permsBuilder.appendCachePermissions("cache", CACHE_CREATE); + return null; + } + }, IgniteException.class, + "CACHE_CREATE should be assigned as system permission, not cache permission" ); permsBuilder @@ -116,7 +128,9 @@ public void testPermissionBuilder() { .appendServicePermissions("service2", SERVICE_INVOKE) .appendServicePermissions("service2", SERVICE_INVOKE) .appendSystemPermissions(ADMIN_VIEW) - .appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE); + .appendSystemPermissions(ADMIN_VIEW, EVENTS_ENABLE) + .appendSystemPermissions(JOIN_AS_SERVER) + .appendSystemPermissions(CACHE_CREATE, CACHE_DESTROY); SecurityPermissionSet actual = permsBuilder.build(); From 9501d0b12fe61517a03b0c1016c09bb9c70d98a1 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Fri, 26 Oct 2018 16:42:39 +0700 Subject: [PATCH 045/403] IGNITE-10016 Web Console: ClusterLoginService should return "Credentials" modal as singleton. (cherry picked from commit 51d8c9f72a0553397d4f471f81f926e498f21114) --- .../agent/components/cluster-login/service.js | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js index 128f3ccc5966f..955d0a3307e6c 100644 --- a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js +++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js @@ -22,6 +22,8 @@ import {CancellationError} from 'app/errors/CancellationError'; export default class ClusterLoginService { static $inject = ['$modal', '$q']; + deferred; + /** * @param {mgcrea.ngStrap.modal.IModalService} $modal * @param {ng.IQService} $q @@ -36,7 +38,12 @@ export default class ClusterLoginService { * @returns {ng.IPromise} */ askCredentials(baseSecrets) { - const deferred = this.$q.defer(); + if (this.deferred) + return this.deferred.promise; + + this.deferred = this.$q.defer(); + + const self = this; const modal = this.$modal({ template: ` @@ -50,11 +57,11 @@ export default class ClusterLoginService { this.secrets = _.clone(baseSecrets); this.onLogin = () => { - deferred.resolve(this.secrets); + self.deferred.resolve(this.secrets); }; this.onHide = () => { - deferred.reject(new CancellationError()); + self.deferred.reject(new CancellationError()); }; }], controllerAs: '$ctrl', @@ -63,7 +70,11 @@ export default class ClusterLoginService { }); return modal.$promise - .then(() => deferred.promise) - .finally(modal.hide); + .then(() => this.deferred.promise) + .finally(() => { + this.deferred = null; + + modal.hide(); + }); } } From 8190d082756fb9cd1e551354a8b24df38dfd604c Mon Sep 17 00:00:00 2001 From: luchnikovnsk Date: Tue, 23 Oct 2018 11:16:40 +0300 Subject: [PATCH 046/403] IGNITE-9719 Extra rebalanceThreadPoolSize check on client node - Fixes #4911. Signed-off-by: Ivan Rakov (cherry picked from commit 962d6a29716195c13519ca48b6d79f3b8541653c) --- .../apache/ignite/internal/IgniteKernal.java | 32 ++++++++------ .../client/ClientConfigurationTest.java | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index c6ec9be1e1848..1f9a5e8f1bfd3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -2581,19 +2581,25 @@ private void ackSpis() { * */ private void ackRebalanceConfiguration() throws IgniteCheckedException { - if (cfg.getSystemThreadPoolSize() <= cfg.getRebalanceThreadPoolSize()) - throw new IgniteCheckedException("Rebalance thread pool size exceed or equals System thread pool size. " + - "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start."); - - if (cfg.getRebalanceThreadPoolSize() < 1) - throw new IgniteCheckedException("Rebalance thread pool size minimal allowed value is 1. " + - "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start."); - - for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) { - if (ccfg.getRebalanceBatchesPrefetchCount() < 1) - throw new IgniteCheckedException("Rebalance batches prefetch count minimal allowed value is 1. " + - "Change CacheConfiguration.rebalanceBatchesPrefetchCount property before next start. " + - "[cache=" + ccfg.getName() + "]"); + if (cfg.isClientMode()) { + if (cfg.getRebalanceThreadPoolSize() != IgniteConfiguration.DFLT_REBALANCE_THREAD_POOL_SIZE) + U.warn(log, "Setting the rebalance pool size has no effect on the client mode"); + } + else { + if (cfg.getSystemThreadPoolSize() <= cfg.getRebalanceThreadPoolSize()) + throw new IgniteCheckedException("Rebalance thread pool size exceed or equals System thread pool size. " + + "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start."); + + if (cfg.getRebalanceThreadPoolSize() < 1) + throw new IgniteCheckedException("Rebalance thread pool size minimal allowed value is 1. " + + "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start."); + + for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) { + if (ccfg.getRebalanceBatchesPrefetchCount() < 1) + throw new IgniteCheckedException("Rebalance batches prefetch count minimal allowed value is 1. " + + "Change CacheConfiguration.rebalanceBatchesPrefetchCount property before next start. " + + "[cache=" + ccfg.getName() + "]"); + } } } diff --git a/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java index bcc212abbe130..287c6ec6793bc 100644 --- a/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java @@ -25,9 +25,19 @@ import java.io.ObjectOutput; import java.io.ObjectOutputStream; import java.util.Collections; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.testframework.GridStringLogger; import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -73,4 +83,36 @@ public void testSerialization() throws IOException, ClassNotFoundException { assertTrue(Comparers.equal(target, desTarget)); } + + /** + * Test check the case when {@link IgniteConfiguration#getRebalanceThreadPoolSize()} is equal to {@link + * IgniteConfiguration#getSystemThreadPoolSize()} + */ + @Test + public void testRebalanceThreadPoolSize() { + GridStringLogger gridStrLog = new GridStringLogger(); + gridStrLog.logLength(1024 * 100); + + IgniteConfiguration cci = Config.getServerConfiguration().setClientMode(true); + cci.setRebalanceThreadPoolSize(cci.getSystemThreadPoolSize()); + cci.setGridLogger(gridStrLog); + + try ( + Ignite si = Ignition.start(Config.getServerConfiguration()); + Ignite ci = Ignition.start(cci)) { + Set collect = si.cluster().nodes().stream() + .filter(new Predicate() { + @Override public boolean test(ClusterNode clusterNode) { + return clusterNode.isClient(); + } + }) + .collect(Collectors.toSet()); + + String log = gridStrLog.toString(); + boolean containsMsg = log.contains("Setting the rebalance pool size has no effect on the client mode"); + + Assert.assertTrue(containsMsg); + Assert.assertEquals(1, collect.size()); + } + } } From 9fe4378588191c33e0f481031c4091ebffc9026a Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Sat, 27 Oct 2018 17:55:54 +0300 Subject: [PATCH 047/403] GNITE-8006 Fixed flaky test --- .../org/apache/ignite/internal/util/IgniteUtilsSelfTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java index 13a104431ce84..ad678b6fc85b1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java @@ -923,7 +923,7 @@ public void testDoInParallelBatch() { fail("Should throw timeout exception"); } catch (Exception e) { - assertTrue(e.getCause() instanceof TimeoutException); + assertTrue(e.toString(), X.hasCause(e, TimeoutException.class)); } } From 6254824e14ef34e5118197dfddb11f55618de909 Mon Sep 17 00:00:00 2001 From: a-polyakov Date: Sat, 27 Oct 2018 14:23:06 +0300 Subject: [PATCH 048/403] IGNITE-10020 Fixed control.sh error comparison method violates its general contract - Fixes #5087. Signed-off-by: Alexey Goncharuk (cherry picked from commit e111d895e7f7f32d6f764683ba8412d61d370455) Signed-off-by: a-polyakov --- .../cache/distribution/CacheDistributionTaskResult.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java index 71de3bbc0dd05..52c6eec308afb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java @@ -294,10 +294,10 @@ public void setUserAttributes(Map userAttrs) { Row other = (Row)o; - int res = grpId - other.grpId; + int res = Integer.compare(grpId, other.grpId); if (res == 0) { - res = partId - other.partId; + res = Integer.compare(partId, other.partId); if (res == 0) res = nodeId.compareTo(other.nodeId); From 8faae6cb6065d08f05d572555dc9b7585be84630 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Mon, 29 Oct 2018 12:14:54 +0300 Subject: [PATCH 049/403] IGNITE-9753 Several optimization of validate_indexes - Fixes #5063. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit 1fe02df8daa94ca0761049cd41659f4ae18f580c) --- .../internal/commandline/CommandHandler.java | 34 ++- .../verify/IndexIntegrityCheckIssue.java | 74 +++++++ .../verify/VisorValidateIndexesJobResult.java | 30 ++- .../resources/META-INF/classnames.properties | 1 + .../visor/verify/ValidateIndexesClosure.java | 205 +++++++++++++++--- .../util/GridCommandHandlerIndexingTest.java | 152 ++++++++----- 6 files changed, 398 insertions(+), 98 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 35981b5fd7e1f..c1853f8531df3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -95,6 +95,7 @@ import org.apache.ignite.internal.visor.tx.VisorTxTask; import org.apache.ignite.internal.visor.tx.VisorTxTaskArg; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; +import org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue; import org.apache.ignite.internal.visor.verify.IndexValidationIssue; import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult; import org.apache.ignite.internal.visor.verify.VisorContentionTask; @@ -780,9 +781,20 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t } } - boolean errors = false; - for (Map.Entry nodeEntry : taskRes.results().entrySet()) { + boolean errors = false; + + log("validate_indexes result on node " + nodeEntry.getKey() + ":"); + + Collection integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures(); + + if (!integrityCheckFailures.isEmpty()) { + errors = true; + + for (IndexIntegrityCheckIssue is : integrityCheckFailures) + log("\t" + is.toString()); + } + Map partRes = nodeEntry.getValue().partitionResult(); for (Map.Entry e : partRes.entrySet()) { @@ -791,10 +803,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t if (!res.issues().isEmpty()) { errors = true; - log(e.getKey().toString() + " " + e.getValue().toString()); + log("\t" + e.getKey().toString() + " " + e.getValue().toString()); for (IndexValidationIssue is : res.issues()) - log(is.toString()); + log("\t\t" + is.toString()); } } @@ -806,18 +818,18 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t if (!res.issues().isEmpty()) { errors = true; - log("SQL Index " + e.getKey() + " " + e.getValue().toString()); + log("\tSQL Index " + e.getKey() + " " + e.getValue().toString()); for (IndexValidationIssue is : res.issues()) - log(is.toString()); + log("\t\t" + is.toString()); } } - } - if (!errors) - log("validate_indexes has finished, no issues found."); - else - log("validate_indexes has finished with errors (listed above)."); + if (!errors) + log("no issues found.\n"); + else + log("issues found (listed above).\n"); + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java new file mode 100644 index 0000000000000..ec6e5b2440765 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.verify; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * + */ +public class IndexIntegrityCheckIssue extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** Cache group name. */ + private String grpName; + + /** T. */ + @GridToStringExclude + private Throwable t; + + /** + * + */ + public IndexIntegrityCheckIssue() { + // Default constructor required for Externalizable. + } + + /** + * @param grpName Group name. + * @param t Data integrity check error. + */ + public IndexIntegrityCheckIssue(String grpName, Throwable t) { + this.grpName = grpName; + this.t = t; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeString(out, this.grpName); + out.writeObject(t); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + this.grpName = U.readString(in); + this.t = (Throwable)in.readObject(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IndexIntegrityCheckIssue.class, this) + ", " + t.getClass() + ": " + t.getMessage(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java index aa74323898bd8..f84fc1a259027 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java @@ -20,11 +20,14 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.util.Collection; +import java.util.Collections; import java.util.Map; import org.apache.ignite.internal.processors.cache.verify.PartitionKey; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; +import org.jetbrains.annotations.NotNull; /** * @@ -39,14 +42,22 @@ public class VisorValidateIndexesJobResult extends VisorDataTransferObject { /** Results of reverse indexes validation from node. */ private Map idxRes; + /** Integrity check issues. */ + private Collection integrityCheckFailures; + /** * @param partRes Results of indexes validation from node. * @param idxRes Results of reverse indexes validation from node. + * @param integrityCheckFailures Collection of indexes integrity check failures. */ - public VisorValidateIndexesJobResult(Map partRes, - Map idxRes) { + public VisorValidateIndexesJobResult( + @NotNull Map partRes, + @NotNull Map idxRes, + @NotNull Collection integrityCheckFailures + ) { this.partRes = partRes; this.idxRes = idxRes; + this.integrityCheckFailures = integrityCheckFailures; } /** @@ -57,7 +68,7 @@ public VisorValidateIndexesJobResult() { /** {@inheritDoc} */ @Override public byte getProtocolVersion() { - return V2; + return V3; } /** @@ -71,13 +82,21 @@ public Map partitionResult() { * @return Results of reverse indexes validation from node. */ public Map indexResult() { - return idxRes; + return idxRes == null ? Collections.emptyMap() : idxRes; + } + + /** + * @return Collection of failed integrity checks. + */ + public Collection integrityCheckFailures() { + return integrityCheckFailures == null ? Collections.emptyList() : integrityCheckFailures; } /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeMap(out, partRes); U.writeMap(out, idxRes); + U.writeCollection(out, integrityCheckFailures); } /** {@inheritDoc} */ @@ -86,6 +105,9 @@ public Map indexResult() { if (protoVer >= V2) idxRes = U.readMap(in); + + if (protoVer >= V3) + integrityCheckFailures = U.readCollection(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index a0e2ba79eba8b..fda0fe469a3f0 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -2206,6 +2206,7 @@ org.apache.ignite.internal.visor.util.VisorEventMapper org.apache.ignite.internal.visor.util.VisorExceptionWrapper org.apache.ignite.internal.visor.util.VisorTaskUtils$4 org.apache.ignite.internal.visor.verify.IndexValidationIssue +org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult org.apache.ignite.internal.visor.verify.VisorContentionJobResult org.apache.ignite.internal.visor.verify.VisorContentionTask diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java index 503b57c494c70..ec02c25c3ca36 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java @@ -18,6 +18,8 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -37,22 +39,29 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; import org.apache.ignite.internal.processors.cache.CacheObjectUtils; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.verify.PartitionKey; import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; import org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; @@ -104,9 +113,15 @@ public class ValidateIndexesClosure implements IgniteCallable> partArgs = new ArrayList<>(); List> idxArgs = new ArrayList<>(); + totalCacheGrps = grpIds.size(); + + Map integrityCheckResults = integrityCheckIndexesPartitions(grpIds); + for (Integer grpId : grpIds) { CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId); - if (grpCtx == null) + if (grpCtx == null || integrityCheckResults.containsKey(grpId)) continue; List parts = grpCtx.topology().localPartitions(); @@ -210,7 +229,8 @@ private VisorValidateIndexesJobResult call0() throws Exception { ArrayList indexes = gridH2Tbl.getIndexes(); for (Index idx : indexes) - idxArgs.add(new T2<>(ctx, idx)); + if (idx instanceof H2TreeIndex) + idxArgs.add(new T2<>(ctx, idx)); } } } @@ -220,15 +240,15 @@ private VisorValidateIndexesJobResult call0() throws Exception { Collections.shuffle(partArgs); Collections.shuffle(idxArgs); + totalPartitions = partArgs.size(); + totalIndexes = idxArgs.size(); + for (T2 t2 : partArgs) procPartFutures.add(processPartitionAsync(t2.get1(), t2.get2())); for (T2 t2 : idxArgs) procIdxFutures.add(processIndexAsync(t2.get1(), t2.get2())); - totalPartitions = procPartFutures.size(); - totalIndexes = procIdxFutures.size(); - Map partResults = new HashMap<>(); Map idxResults = new HashMap<>(); @@ -250,6 +270,9 @@ private VisorValidateIndexesJobResult call0() throws Exception { idxResults.putAll(idxRes); } + + log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + + totalIndexes + " indexes."); } catch (InterruptedException | ExecutionException e) { for (int j = curPart; j < procPartFutures.size(); j++) @@ -258,15 +281,102 @@ private VisorValidateIndexesJobResult call0() throws Exception { for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false); - if (e instanceof InterruptedException) - throw new IgniteInterruptedException((InterruptedException)e); - else if (e.getCause() instanceof IgniteException) - throw (IgniteException)e.getCause(); - else - throw new IgniteException(e.getCause()); + throw unwrapFutureException(e); + } + + return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values()); + } + + /** + * @param grpIds Group ids. + */ + private Map integrityCheckIndexesPartitions(Set grpIds) { + List>> integrityCheckFutures = new ArrayList<>(grpIds.size()); + + for (Integer grpId: grpIds) { + final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId); + + if (grpCtx == null) { + integrityCheckedIndexes.incrementAndGet(); + + continue; + } + + Future> checkFut = + calcExecutor.submit(new Callable>() { + @Override public T2 call() throws Exception { + IndexIntegrityCheckIssue issue = integrityCheckIndexPartition(grpCtx); + + return new T2<>(grpCtx.groupId(), issue); + } + }); + + integrityCheckFutures.add(checkFut); + } + + Map integrityCheckResults = new HashMap<>(); + + int curFut = 0; + try { + for (Future> fut : integrityCheckFutures) { + T2 res = fut.get(); + + if (res.getValue() != null) + integrityCheckResults.put(res.getKey(), res.getValue()); + } + } + catch (InterruptedException | ExecutionException e) { + for (int j = curFut; j < integrityCheckFutures.size(); j++) + integrityCheckFutures.get(j).cancel(false); + + throw unwrapFutureException(e); + } + + return integrityCheckResults; + } + + /** + * @param gctx Cache group context. + */ + private IndexIntegrityCheckIssue integrityCheckIndexPartition(CacheGroupContext gctx) { + GridKernalContext ctx = ignite.context(); + GridCacheSharedContext cctx = ctx.cache().context(); + + try { + FilePageStoreManager pageStoreMgr = (FilePageStoreManager)cctx.pageStore(); + + if (pageStoreMgr == null) + return null; + + int pageSz = gctx.dataRegion().pageMemory().pageSize(); + + PageStore pageStore = pageStoreMgr.getStore(gctx.groupId(), PageIdAllocator.INDEX_PARTITION); + + long pageId = PageIdUtils.pageId(PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX, 0); + + ByteBuffer buf = ByteBuffer.allocateDirect(pageSz); + + buf.order(ByteOrder.nativeOrder()); + + for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) { + buf.clear(); + + pageStore.read(pageId, buf, true); + } + + return null; } + catch (Throwable t) { + log.error("Integrity check of index partition of cache group " + gctx.cacheOrGroupName() + " failed", t); + + return new IndexIntegrityCheckIssue(gctx.cacheOrGroupName(), t); + } + finally { + integrityCheckedIndexes.incrementAndGet(); - return new VisorValidateIndexesJobResult(partResults, idxResults); + printProgressIfNeeded("Current progress of ValidateIndexesClosure: checked integrity of " + + integrityCheckedIndexes.get() + " index partitions of " + totalCacheGrps + " cache groups"); + } } /** @@ -371,6 +481,19 @@ else if (current++ % checkThrough > 0) if (cacheCtx == null) throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId); + if (row.link() == 0L) { + String errMsg = "Invalid partition row, possibly deleted"; + + log.error(errMsg); + + IndexValidationIssue is = new IndexValidationIssue(null, cacheCtx.name(), null, + new IgniteCheckedException(errMsg)); + + enoughIssues |= partRes.reportIssue(is); + + continue; + } + try { QueryTypeDescriptorImpl res = (QueryTypeDescriptorImpl)m.invoke( qryProcessor, cacheCtx.name(), cacheCtx.cacheObjectContext(), row.key(), row.value(), true); @@ -392,6 +515,9 @@ else if (current++ % checkThrough > 0) ArrayList indexes = gridH2Tbl.getIndexes(); for (Index idx : indexes) { + if (!(idx instanceof H2TreeIndex)) + continue; + try { Cursor cursor = idx.find((Session) null, h2Row, h2Row); @@ -434,7 +560,7 @@ else if (current++ % checkThrough > 0) finally { part.release(); - printProgressIfNeeded(); + printProgressOfIndexValidationIfNeeded(); } PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName()); @@ -447,16 +573,21 @@ else if (current++ % checkThrough > 0) /** * */ - private void printProgressIfNeeded() { - long curTs = U.currentTimeMillis(); + private void printProgressOfIndexValidationIfNeeded() { + printProgressIfNeeded("Current progress of ValidateIndexesClosure: processed " + + processedPartitions.get() + " of " + totalPartitions + " partitions, " + + processedIndexes.get() + " of " + totalIndexes + " SQL indexes"); + } + /** + * + */ + private void printProgressIfNeeded(String msg) { + long curTs = U.currentTimeMillis(); long lastTs = lastProgressPrintTs.get(); - if (curTs - lastTs >= 60_000 && lastProgressPrintTs.compareAndSet(lastTs, curTs)) { - log.warning("Current progress of ValidateIndexesClosure: processed " + - processedPartitions.get() + " of " + totalPartitions + " partitions, " + - processedIndexes.get() + " of " + totalIndexes + " SQL indexes"); - } + if (curTs - lastTs >= 60_000 && lastProgressPrintTs.compareAndSet(lastTs, curTs)) + log.warning(msg); } /** @@ -546,12 +677,14 @@ else if (current++ % checkThrough > 0) h2key = h2Row.key(); - CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key); + if (h2Row.link() != 0L) { + CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key); - if (cacheDataStoreRow == null) - throw new IgniteCheckedException("Key is present in SQL index, but can't be found in CacheDataTree."); - - previousKey = h2key; + if (cacheDataStoreRow == null) + throw new IgniteCheckedException("Key is present in SQL index, but can't be found in CacheDataTree."); + } + else + throw new IgniteCheckedException("Invalid index row, possibly deleted " + h2Row); } catch (Throwable t) { Object o = CacheObjectUtils.unwrapBinaryIfNeeded( @@ -564,14 +697,34 @@ else if (current++ % checkThrough > 0) enoughIssues |= idxValidationRes.reportIssue(is); } + finally { + previousKey = h2key; + } } String uniqueIdxName = "[cache=" + ctx.name() + ", idx=" + idx.getName() + "]"; processedIndexes.incrementAndGet(); - printProgressIfNeeded(); + printProgressOfIndexValidationIfNeeded(); return Collections.singletonMap(uniqueIdxName, idxValidationRes); } + + /** + * @param e Future result exception. + * @return Unwrapped exception. + */ + private IgniteException unwrapFutureException(Exception e) { + assert e instanceof InterruptedException || e instanceof ExecutionException : "Expecting either InterruptedException " + + "or ExecutionException"; + + if (e instanceof InterruptedException) + return new IgniteInterruptedException((InterruptedException)e); + else if (e.getCause() instanceof IgniteException) + return (IgniteException)e.getCause(); + else + return new IgniteException(e.getCause()); + } + } diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java index 48e94c1e62cce..c7693d214b0ac 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java @@ -17,6 +17,9 @@ package org.apache.ignite.util; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; import java.io.Serializable; import java.util.ArrayList; import java.util.Iterator; @@ -24,8 +27,8 @@ import java.util.concurrent.ThreadLocalRandom; import javax.cache.Cache; import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.QueryEntity; @@ -39,6 +42,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.tree.SearchRow; import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.util.lang.GridIterator; @@ -47,57 +51,35 @@ import org.apache.ignite.internal.util.typedef.internal.U; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME; /** * */ public class GridCommandHandlerIndexingTest extends GridCommandHandlerTest { + /** Test cache name. */ + private static final String CACHE_NAME = "persons-cache-vi"; + /** * Tests that validation doesn't fail if nothing is broken. */ public void testValidateIndexesNoErrors() throws Exception { - Ignite ignite = startGrids(2); - - ignite.cluster().active(true); - - Ignite client = startGrid("client"); - - String cacheName = "persons-cache-vi"; - - IgniteCache personCache = createPersonCache(client, cacheName); - - ThreadLocalRandom rand = ThreadLocalRandom.current(); - - for (int i = 0; i < 10_000; i++) - personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong()))); + prepareGridForTest(); injectTestSystemOut(); - assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName)); + assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", CACHE_NAME)); - assertTrue(testOut.toString().contains("validate_indexes has finished, no issues found")); + assertTrue(testOut.toString().contains("no issues found")); } /** * Tests that missing rows in CacheDataTree are detected. */ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception { - Ignite ignite = startGrids(2); + Ignite ignite = prepareGridForTest(); - ignite.cluster().active(true); - - Ignite client = startGrid("client"); - - String cacheName = "persons-cache-vi"; - - IgniteCache personCache = createPersonCache(client, cacheName); - - ThreadLocalRandom rand = ThreadLocalRandom.current(); - - for (int i = 0; i < 10_000; i++) - personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong()))); - - breakCacheDataTree(ignite, cacheName, 1); + breakCacheDataTree(ignite, CACHE_NAME, 1); injectTestSystemOut(); @@ -105,11 +87,11 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception { execute( "--cache", "validate_indexes", - cacheName, + CACHE_NAME, "checkFirst", "10000", "checkThrough", "10")); - assertTrue(testOut.toString().contains("validate_indexes has finished with errors")); + assertTrue(testOut.toString().contains("issues found (listed above)")); assertTrue(testOut.toString().contains( "Key is present in SQL index, but is missing in corresponding data page.")); @@ -119,6 +101,46 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception { * Tests that missing rows in H2 indexes are detected. */ public void testBrokenSqlIndexShouldFailValidation() throws Exception { + Ignite ignite = prepareGridForTest(); + + breakSqlIndex(ignite, CACHE_NAME); + + injectTestSystemOut(); + + assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", CACHE_NAME)); + + assertTrue(testOut.toString().contains("issues found (listed above)")); + } + + /** + * Tests that missing rows in H2 indexes are detected. + */ + public void testCorruptedIndexPartitionShouldFailValidation() throws Exception { + Ignite ignite = prepareGridForTest(); + + forceCheckpoint(); + + File idxPath = indexPartition(ignite, CACHE_NAME); + + stopAllGrids(); + + corruptIndexPartition(idxPath); + + startGrids(2); + + awaitPartitionMapExchange(); + + injectTestSystemOut(); + + assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", CACHE_NAME)); + + assertTrue(testOut.toString().contains("issues found (listed above)")); + } + + /** + * + */ + private Ignite prepareGridForTest() throws Exception{ Ignite ignite = startGrids(2); ignite.cluster().active(true); @@ -127,20 +149,52 @@ public void testBrokenSqlIndexShouldFailValidation() throws Exception { String cacheName = "persons-cache-vi"; - IgniteCache personCache = createPersonCache(client, cacheName); + client.getOrCreateCache(new CacheConfiguration() + .setName(cacheName) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setBackups(1) + .setQueryEntities(F.asList(personEntity(true, true))) + .setAffinity(new RendezvousAffinityFunction(false, 32))); ThreadLocalRandom rand = ThreadLocalRandom.current(); - for (int i = 0; i < 10_000; i++) - personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong()))); + try (IgniteDataStreamer streamer = client.dataStreamer(CACHE_NAME);) { + for (int i = 0; i < 10_000; i++) + streamer.addData(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong()))); + } - breakSqlIndex(ignite, cacheName); + return ignite; + } - injectTestSystemOut(); + /** + * Get index partition file for specific node and cache. + */ + private File indexPartition(Ignite ig, String cacheName) { + IgniteEx ig0 = (IgniteEx)ig; - assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName)); + FilePageStoreManager pageStoreManager = ((FilePageStoreManager) ig0.context().cache().context().pageStore()); - assertTrue(testOut.toString().contains("validate_indexes has finished with errors")); + return new File(pageStoreManager.cacheWorkDir(false, cacheName), INDEX_FILE_NAME); + } + + /** + * Write some random trash in index partition. + */ + private void corruptIndexPartition(File path) throws IOException { + assertTrue(path.exists()); + + ThreadLocalRandom rand = ThreadLocalRandom.current(); + + try (RandomAccessFile idx = new RandomAccessFile(path, "rw")) { + byte[] trash = new byte[1024]; + + rand.nextBytes(trash); + + idx.seek(4096); + + idx.write(trash); + } } /** @@ -241,22 +295,6 @@ private void breakSqlIndex(Ignite ig, String cacheName) throws Exception { } } - /** - * Dynamically creates cache with SQL indexes. - * - * @param ig Client. - * @param cacheName Cache name. - */ - private IgniteCache createPersonCache(Ignite ig, String cacheName) { - return ig.getOrCreateCache(new CacheConfiguration() - .setName(cacheName) - .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) - .setAtomicityMode(CacheAtomicityMode.ATOMIC) - .setBackups(1) - .setQueryEntities(F.asList(personEntity(true, true))) - .setAffinity(new RendezvousAffinityFunction(false, 32))); - } - /** * @param idxName Index name. * @param idxOrgId Index org id. From 64227a8154787275f01c1c31252db5159f5981ee Mon Sep 17 00:00:00 2001 From: Oleg Ostanin Date: Mon, 29 Oct 2018 13:28:07 +0300 Subject: [PATCH 050/403] IGNITE-9682 Update partition full map in parallel - Fixes #4824. Signed-off-by: Alexey Goncharuk --- .../GridDhtPartitionsExchangeFuture.java | 66 +++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 63d36354327a6..92effb1b495be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -133,6 +133,7 @@ import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverJoinEvent; import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverLeftEvent; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE; +import static org.apache.ignite.internal.util.IgniteUtils.doInParallel; /** * Future for exchanging partition maps. @@ -3983,39 +3984,50 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa long time = System.currentTimeMillis(); - for (Map.Entry entry : msg.partitions().entrySet()) { - Integer grpId = entry.getKey(); + int parallelismLvl = cctx.kernalContext().config().getSystemThreadPoolSize(); - CacheGroupContext grp = cctx.cache().cacheGroup(grpId); + // Reserve at least 2 threads for system operations. + parallelismLvl = Math.max(1, parallelismLvl - 2); - if (grp != null) { - CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId, - grp.topology().partitions()); + try { + doInParallel( + parallelismLvl, + cctx.kernalContext().getSystemExecutorService(), + msg.partitions().keySet(), grpId -> { + CacheGroupContext grp = cctx.cache().cacheGroup(grpId); - grp.topology().update(resTopVer, - entry.getValue(), - cntrMap, - msg.partsToReload(cctx.localNodeId(), grpId), - msg.partitionSizes(grpId), - null); - } - else { - ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE); + if (grp != null) { + CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId, + grp.topology().partitions()); - if (oldest != null && oldest.isLocal()) { - GridDhtPartitionTopology top = cctx.exchange().clientTopology(grpId, events().discoveryCache()); + grp.topology().update(resTopVer, + msg.partitions().get(grpId), + cntrMap, + msg.partsToReload(cctx.localNodeId(), grpId), + msg.partitionSizes(grpId), + null); + } + else { + ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE); - CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId, - top.partitions()); + if (oldest != null && oldest.isLocal()) { + GridDhtPartitionTopology top = cctx.exchange().clientTopology(grpId, events().discoveryCache()); - top.update(resTopVer, - entry.getValue(), - cntrMap, - Collections.emptySet(), - null, - null); - } - } + CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId, + top.partitions()); + + top.update(resTopVer, + msg.partitions().get(grpId), + cntrMap, + Collections.emptySet(), + null, + null); + } + } + }); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); } partitionsReceived = true; From 981fce9188f27b3061dcf7f6875f5da15c13ec8d Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 29 Oct 2018 12:35:30 +0300 Subject: [PATCH 051/403] IGNITE-9790 Fixed delayed full message handling in case of changed coordinator. - Fixes #5190. Signed-off-by: Alexey Goncharuk (cherry picked from commit 2906a16) --- .../GridDhtPartitionsExchangeFuture.java | 14 ++-- .../GridDhtPartitionsFullMessage.java | 16 ++++- ...itionsExchangeCoordinatorFailoverTest.java | 64 +++++++++++++++++++ 3 files changed, 85 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 63d36354327a6..d60a017322947 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -184,7 +184,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte private List srvNodes; /** */ - private ClusterNode crd; + private volatile ClusterNode crd; /** ExchangeFuture id. */ private final GridDhtPartitionExchangeId exchId; @@ -4585,12 +4585,12 @@ public boolean reconnectOnError(Throwable e) { * @return {@code True} If partition changes triggered by receiving Single/Full messages are not finished yet. */ public boolean partitionChangesInProgress() { - boolean isCoordinator = crd.equals(cctx.localNode()); + ClusterNode crd0 = crd; - if (isCoordinator) - return !partitionsSent; - else - return !partitionsReceived; + if (crd0 == null) + return false; + + return crd0.equals(cctx.localNode()) ? !partitionsSent : !partitionsReceived; } /** @@ -4624,7 +4624,7 @@ public synchronized boolean addOrMergeDelayedFullMessage(ClusterNode node, GridD }); } else - delayedLatestMsg.merge(fullMsg); + delayedLatestMsg.merge(fullMsg, cctx.discovery()); return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index b84dc79e821f3..a63ab70e28d53 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -25,8 +25,10 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridDirectMap; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; @@ -803,7 +805,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { * * @param other Other full message. */ - public void merge(GridDhtPartitionsFullMessage other) { + public void merge(GridDhtPartitionsFullMessage other, GridDiscoveryManager discovery) { assert other.exchangeId() == null && exchangeId() == null : "Both current and merge full message must have exchangeId == null" + other.exchangeId() + "," + exchangeId(); @@ -814,8 +816,18 @@ public void merge(GridDhtPartitionsFullMessage other) { GridDhtPartitionFullMap currMap = partitions().get(grpId); - if (currMap == null || updMap.compareTo(currMap) >= 0) + if (currMap == null) partitions().put(grpId, updMap); + else { + ClusterNode currentMapSentBy = discovery.node(currMap.nodeId()); + ClusterNode newMapSentBy = discovery.node(updMap.nodeId()); + + if (newMapSentBy == null) + return; + + if (currentMapSentBy == null || newMapSentBy.order() > currentMapSentBy.order() || updMap.compareTo(currMap) >= 0) + partitions().put(grpId, updMap); + } } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java index a2adcf77a7bac..1c847f9fea843 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java @@ -152,4 +152,68 @@ public void testNewCoordinatorCompletedExchange() throws Exception { cache.put(0, 0); } } + + /** + * Test checks that delayed full messages are processed correctly in case of changed coordinator. + * + * @throws Exception If failed. + */ + public void testDelayedFullMessageReplacedIfCoordinatorChanged() throws Exception { + IgniteEx crd = startGrid("crd"); + + IgniteEx newCrd = startGrid(1); + + IgniteEx problemNode = startGrid(2); + + crd.cluster().active(true); + + awaitPartitionMapExchange(); + + blockSendingFullMessage(crd, problemNode); + + IgniteInternalFuture joinNextNodeFut = GridTestUtils.runAsync(() -> startGrid(3)); + + joinNextNodeFut.get(); + + U.sleep(5000); + + blockSendingFullMessage(newCrd, problemNode); + + IgniteInternalFuture stopCoordinatorFut = GridTestUtils.runAsync(() -> stopGrid("crd")); + + stopCoordinatorFut.get(); + + U.sleep(5000); + + TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(newCrd); + + spi.stopBlock(true); + + awaitPartitionMapExchange(); + } + + /** + * Blocks sending full message from coordinator to non-coordinator node. + * @param from Coordinator node. + * @param to Non-coordinator node. + */ + private void blockSendingFullMessage(IgniteEx from, IgniteEx to) { + // Block FullMessage for newly joined nodes. + TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(from); + + // Delay sending full messages (without exchange id). + spi.blockMessages((node, msg) -> { + if (msg instanceof GridDhtPartitionsFullMessage) { + GridDhtPartitionsFullMessage fullMsg = (GridDhtPartitionsFullMessage) msg; + + if (fullMsg.exchangeId() != null && node.order() == to.localNode().order()) { + log.warning("Blocked sending " + msg + " to " + to.localNode()); + + return true; + } + } + + return false; + }); + } } From 3ed5599fd0492c827b908e46da698951fa334b08 Mon Sep 17 00:00:00 2001 From: ekaterina-nbl Date: Mon, 29 Oct 2018 15:49:33 +0300 Subject: [PATCH 052/403] IGNITE-10022: JS, PHP thin clients: a more meaningful exception when ENUM type is not registered This closes #5187 (cherry picked from commit 2ad04930dd5854c4272fb347858292e464daa83d) --- modules/platforms/nodejs/lib/EnumItem.js | 17 ++- modules/platforms/nodejs/lib/Errors.js | 12 ++ .../nodejs/lib/internal/BinaryUtils.js | 4 + .../platforms/nodejs/spec/TestingHelper.js | 7 ++ .../spec/cache/CachePutGetDiffTypes.spec.js | 39 ++++++ .../Internal/Binary/BinaryCommunicator.php | 27 ++-- .../Ignite/Internal/Binary/BinaryUtils.php | 12 ++ .../platforms/php/tests/CachePutGetTest.php | 119 ++++++++++++++++++ 8 files changed, 221 insertions(+), 16 deletions(-) diff --git a/modules/platforms/nodejs/lib/EnumItem.js b/modules/platforms/nodejs/lib/EnumItem.js index 1d1725e3d5908..5e80da9dee2b5 100644 --- a/modules/platforms/nodejs/lib/EnumItem.js +++ b/modules/platforms/nodejs/lib/EnumItem.js @@ -17,6 +17,7 @@ 'use strict'; +const Util = require('util'); const ArgumentChecker = require('./internal/ArgumentChecker'); const Errors = require('./Errors'); @@ -157,14 +158,18 @@ class EnumItem { * @ignore */ async _write(communicator, buffer) { + const type = await this._getType(communicator, this._typeId); + if (!type || !type._isEnum) { + throw Errors.IgniteClientError.enumSerializationError( + true, Util.format('enum type id "%d" is not registered', this._typeId)); + } buffer.writeInteger(this._typeId); if (this._ordinal !== null) { buffer.writeInteger(this._ordinal); return; } else if (this._name !== null || this._value !== null) { - const type = await this._getType(communicator, this._typeId); - if (type._isEnum && type._enumValues) { + if (type._enumValues) { for (let i = 0; i < type._enumValues.length; i++) { if (this._name === type._enumValues[i][0] || this._value === type._enumValues[i][1]) { @@ -185,8 +190,12 @@ class EnumItem { this._typeId = buffer.readInteger(); this._ordinal = buffer.readInteger(); const type = await this._getType(communicator, this._typeId); - if (!type._isEnum || !type._enumValues || type._enumValues.length <= this._ordinal) { - throw new Errors.IgniteClientError('EnumItem can not be deserialized: type mismatch'); + if (!type || !type._isEnum) { + throw Errors.IgniteClientError.enumSerializationError( + false, Util.format('enum type id "%d" is not registered', this._typeId)); + } + else if (!type._enumValues || type._enumValues.length <= this._ordinal) { + throw Errors.IgniteClientError.enumSerializationError(false, 'type mismatch'); } this._name = type._enumValues[this._ordinal][0]; this._value = type._enumValues[this._ordinal][1]; diff --git a/modules/platforms/nodejs/lib/Errors.js b/modules/platforms/nodejs/lib/Errors.js index 57a7a8c291306..89baf386ca0e8 100644 --- a/modules/platforms/nodejs/lib/Errors.js +++ b/modules/platforms/nodejs/lib/Errors.js @@ -83,6 +83,18 @@ class IgniteClientError extends Error { } return new IgniteClientError(msg); } + + /** + * EnumItem serialization/deserialization errors. + * @ignore + */ + static enumSerializationError(serialize, message = null) { + let msg = serialize ? 'Enum item can not be serialized' : 'Enum item can not be deserialized'; + if (message) { + msg = msg + ': ' + message; + } + return new IgniteClientError(msg); + } } /** diff --git a/modules/platforms/nodejs/lib/internal/BinaryUtils.js b/modules/platforms/nodejs/lib/internal/BinaryUtils.js index 2619df7961fd7..fe1e4034a9958 100644 --- a/modules/platforms/nodejs/lib/internal/BinaryUtils.js +++ b/modules/platforms/nodejs/lib/internal/BinaryUtils.js @@ -497,6 +497,10 @@ class BinaryUtils { expectedTypeCode === BinaryUtils.TYPE_CODE.COMPLEX_OBJECT) { return; } + else if (expectedTypeCode === BinaryUtils.TYPE_CODE.ENUM && + actualTypeCode === BinaryUtils.TYPE_CODE.BINARY_ENUM) { + return; + } else if (actualTypeCode !== expectedTypeCode) { throw Errors.IgniteClientError.typeCastError(actualTypeCode, expectedTypeCode); } diff --git a/modules/platforms/nodejs/spec/TestingHelper.js b/modules/platforms/nodejs/spec/TestingHelper.js index 79a53361dce8e..78df0cb37915f 100644 --- a/modules/platforms/nodejs/spec/TestingHelper.js +++ b/modules/platforms/nodejs/spec/TestingHelper.js @@ -232,6 +232,13 @@ class TestingHelper { TestingHelper.checkError(error, Errors.IgniteClientError, done) } + static checkEnumItemSerializationError(error, done) { + if (!(error instanceof Errors.IgniteClientError) || + error.message.indexOf('Enum item can not be serialized') < 0) { + done.fail('unexpected error: ' + error); + } + } + static checkError(error, errorType, done) { if (!(error instanceof errorType)) { done.fail('unexpected error: ' + error); diff --git a/modules/platforms/nodejs/spec/cache/CachePutGetDiffTypes.spec.js b/modules/platforms/nodejs/spec/cache/CachePutGetDiffTypes.spec.js index 28a9ae3825f43..a6e1bba8c64a6 100644 --- a/modules/platforms/nodejs/spec/cache/CachePutGetDiffTypes.spec.js +++ b/modules/platforms/nodejs/spec/cache/CachePutGetDiffTypes.spec.js @@ -543,6 +543,45 @@ describe('cache put get test suite >', () => { catch(error => done.fail(error)); }); + it('put enum items', (done) => { + Promise.resolve(). + then(async () => { + const fakeTypeId = 12345; + const enumItem1 = new EnumItem(fakeTypeId); + enumItem1.setOrdinal(1); + await putEnumItem(enumItem1, null, done); + await putEnumItem(enumItem1, ObjectType.PRIMITIVE_TYPE.ENUM, done); + const enumItem2 = new EnumItem(fakeTypeId); + enumItem2.setName('name'); + await putEnumItem(enumItem2, null, done); + await putEnumItem(enumItem2, ObjectType.PRIMITIVE_TYPE.ENUM, done); + const enumItem3 = new EnumItem(fakeTypeId); + enumItem3.setValue(2); + await putEnumItem(enumItem3, null, done); + await putEnumItem(enumItem3, ObjectType.PRIMITIVE_TYPE.ENUM, done); + }). + then(done). + catch(error => done.fail(error)); + }); + + async function putEnumItem(value, valueType, done) { + const cache = igniteClient.getCache(CACHE_NAME). + setKeyType(null). + setValueType(valueType); + const key = new Date(); + // Enums registration is not supported by the client, therefore put EnumItem must throw IgniteClientError + try { + await cache.put(key, value); + done.fail('put EnumItem must throw IgniteClientError'); + } + catch (err) { + TestingHelper.checkEnumItemSerializationError(err, done); + } + finally { + await cache.removeAll(); + } + } + async function putGetPrimitiveValues(keyType, valueType, key, value, modificator) { const cache = await igniteClient.getCache(CACHE_NAME). setKeyType(keyType). diff --git a/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryCommunicator.php b/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryCommunicator.php index 520063c65f524..781a73073d0d1 100644 --- a/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryCommunicator.php +++ b/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryCommunicator.php @@ -265,8 +265,10 @@ private function readEnum(MessageBuffer $buffer): EnumItem $ordinal = $buffer->readInteger(); $enumItem->setOrdinal($ordinal); $type = $this->typeStorage->getType($enumItem->getTypeId()); - if (!$type->isEnum() || !$type->getEnumValues() || count($type->getEnumValues()) <= $ordinal) { - BinaryUtils::serializationError(false, 'EnumItem can not be deserialized: type mismatch'); + if (!$type || !$type->isEnum()) { + BinaryUtils::enumSerializationError(false, sprintf('enum type id "%d" is not registered', $enumItem->getTypeId())); + } elseif (!$type->getEnumValues() || count($type->getEnumValues()) <= $ordinal) { + BinaryUtils::enumSerializationError(false, 'type mismatch'); } $enumValues = $type->getEnumValues(); $enumItem->setName($enumValues[$ordinal][0]); @@ -396,21 +398,22 @@ private function writeUUID(MessageBuffer $buffer, array $value): void private function writeEnum(MessageBuffer $buffer, EnumItem $enumValue): void { + $type = $this->typeStorage->getType($enumValue->getTypeId()); + if (!$type || !$type->isEnum()) { + BinaryUtils::enumSerializationError(true, sprintf('enum type id "%d" is not registered', $enumValue->getTypeId())); + } $buffer->writeInteger($enumValue->getTypeId()); if ($enumValue->getOrdinal() !== null) { $buffer->writeInteger($enumValue->getOrdinal()); return; } elseif ($enumValue->getName() !== null || $enumValue->getValue() !== null) { - $type = $this->typeStorage->getType($enumValue->getTypeId()); - if ($type && $type->isEnum()) { - $enumValues = $type->getEnumValues(); - if ($enumValues) { - for ($i = 0; $i < count($enumValues); $i++) { - if ($enumValue->getName() === $enumValues[$i][0] || - $enumValue->getValue() === $enumValues[$i][1]) { - $buffer->writeInteger($i); - return; - } + $enumValues = $type->getEnumValues(); + if ($enumValues) { + for ($i = 0; $i < count($enumValues); $i++) { + if ($enumValue->getName() === $enumValues[$i][0] || + $enumValue->getValue() === $enumValues[$i][1]) { + $buffer->writeInteger($i); + return; } } } diff --git a/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryUtils.php b/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryUtils.php index ad0bf56c77f19..e9ff2f1b98b04 100644 --- a/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryUtils.php +++ b/modules/platforms/php/src/Apache/Ignite/Internal/Binary/BinaryUtils.php @@ -203,6 +203,9 @@ public static function checkTypesCompatibility($expectedType, int $actualTypeCod $actualTypeCode === ObjectType::BINARY_OBJECT && $expectedTypeCode === ObjectType::COMPLEX_OBJECT) { return; + } elseif ($expectedTypeCode === ObjectType::ENUM && + $actualTypeCode === ObjectType::BINARY_ENUM) { + return; } elseif ($actualTypeCode !== $expectedTypeCode) { BinaryUtils::typeCastError($actualTypeCode, $expectedTypeCode); } @@ -419,6 +422,15 @@ public static function serializationError(bool $serialize, string $message = nul throw new ClientException($msg); } + public static function enumSerializationError(bool $serialize, string $message = null): void + { + $msg = $serialize ? 'Enum item can not be serialized' : 'Enum item can not be deserialized'; + if ($message) { + $msg = $msg . ': ' . $message; + } + throw new ClientException($msg); + } + public static function typeCastError($fromType, $toType): void { throw new ClientException(sprintf('Type "%s" can not be cast to %s', diff --git a/modules/platforms/php/tests/CachePutGetTest.php b/modules/platforms/php/tests/CachePutGetTest.php index 9d15ab2810564..c3ff1e9f7de0b 100644 --- a/modules/platforms/php/tests/CachePutGetTest.php +++ b/modules/platforms/php/tests/CachePutGetTest.php @@ -18,14 +18,20 @@ namespace Apache\Ignite\Tests; +use \DateTime; use Ds\Map; use Ds\Set; use PHPUnit\Framework\TestCase; +use Apache\Ignite\Type\ObjectType; use Apache\Ignite\Type\MapObjectType; use Apache\Ignite\Type\CollectionObjectType; use Apache\Ignite\Type\ObjectArrayType; use Apache\Ignite\Type\ComplexObjectType; use Apache\Ignite\Data\BinaryObject; +use Apache\Ignite\Data\Date; +use Apache\Ignite\Data\Timestamp; +use Apache\Ignite\Data\EnumItem; +use Apache\Ignite\Exception\ClientException; class TstComplObjectWithPrimitiveFields { @@ -490,6 +496,119 @@ public function testPutGetObjectArrayOfObjectArraysOfComplexObjectsWithDefaultFi $this->putGetObjectArrays(new ObjectArrayType(new ObjectArrayType(new ComplexObjectType())), $array); } + public function testPutGetDateTime(): void + { + $this->putGetDate("Y-m-d H:i:s", "2018-10-19 18:31:13", 0); + $this->putGetDate("Y-m-d H:i:s", "2018-10-19 18:31:13", 29726); + $this->putGetDate("Y-m-d H:i:s", "2018-10-19 18:31:13", 999999); + + $this->putGetTimestamp("Y-m-d H:i:s", "2018-10-19 18:31:13", 0); + $this->putGetTimestamp("Y-m-d H:i:s", "2018-10-19 18:31:13", 29726000); + $this->putGetTimestamp("Y-m-d H:i:s", "2018-10-19 18:31:13", 999999999); + + $this->putGetTimestampFromDateTime("Y-m-d H:i:s", "2018-10-19 18:31:13", 0); + $this->putGetTimestampFromDateTime("Y-m-d H:i:s", "2018-10-19 18:31:13", 29726); + $this->putGetTimestampFromDateTime("Y-m-d H:i:s", "2018-10-19 18:31:13", 999999); + } + + public function testPutEnumItems(): void + { + $fakeTypeId = 12345; + $enumItem1 = new EnumItem($fakeTypeId); + $enumItem1->setOrdinal(1); + $this->putEnumItem($enumItem1, null); + $this->putEnumItem($enumItem1, ObjectType::ENUM); + $enumItem2 = new EnumItem($fakeTypeId); + $enumItem2->setName('name'); + $this->putEnumItem($enumItem2, null); + $this->putEnumItem($enumItem2, ObjectType::ENUM); + $enumItem3 = new EnumItem($fakeTypeId); + $enumItem3->setOrdinal(2); + $this->putEnumItem($enumItem3, null); + $this->putEnumItem($enumItem3, ObjectType::ENUM); + } + + private function putEnumItem($value, $valueType): void + { + $key = microtime(); + self::$cache-> + setKeyType(null)-> + setValueType($valueType); + // Enums registration is not supported by the client, therefore put EnumItem must throw ClientException + try { + self::$cache->put($key, $value); + $this->fail('put EnumItem must throw ClientException'); + } catch (ClientException $e) { + $this->assertContains('Enum item can not be serialized', $e->getMessage()); + } finally { + self::$cache->removeAll(); + } + } + + private function putGetDate(string $format, string $dateString, int $micros): void + { + $key = microtime(); + self::$cache-> + setKeyType(null)-> + setValueType(ObjectType::DATE); + try { + $dt = DateTime::createFromFormat("$format.u", sprintf("%s.%06d", $dateString, $micros)); + $iDate = Date::fromDateTime($dt); + self::$cache->put($key, $iDate); + $result = self::$cache->get($key); + + $this->assertEquals(sprintf("%06d", intval($micros / 1000) * 1000), $result->toDateTime()->format('u')); + $this->assertEquals($dateString, $result->toDateTime()->format($format)); + } finally { + self::$cache->removeAll(); + } + } + + private function putGetTimestamp(string $format, string $dateString, int $nanos): void + { + $key = microtime(); + self::$cache-> + setKeyType(null)-> + setValueType(ObjectType::TIMESTAMP); + + try { + $millis = intval($nanos / 1000000); + $nanosInMillis = $nanos % 1000000; + self::$cache->put($key, + new Timestamp( + DateTime::createFromFormat($format, $dateString)->getTimestamp() * 1000 + $millis, + $nanosInMillis + ) + ); + $result = self::$cache->get($key); + + $this->assertEquals($nanos % 1000000, $result->getNanos()); + $this->assertEquals($dateString, $result->toDateTime()->format($format)); + } finally { + self::$cache->removeAll(); + } + } + + private function putGetTimestampFromDateTime(string $format, string $dateString, $micros): void + { + $key = microtime(); + self::$cache-> + setKeyType(null)-> + setValueType(ObjectType::TIMESTAMP); + + try { + self::$cache->put($key, Timestamp::fromDateTime( + DateTime::createFromFormat("$format.u", sprintf("%s.%06d", $dateString, $micros)) + )); + $result = self::$cache->get($key); + + $this->assertEquals(intval($micros / 1000) * 1000, $result->toDateTime()->format('u')); + $this->assertEquals($dateString, $result->toDateTime()->format($format)); + } finally { + self::$cache->removeAll(); + } + } + private function putGetObjectArrays(?ObjectArrayType $arrayType, array $value): void { $key = microtime(); From 39a144ce68f17288fa30706134a213502487800f Mon Sep 17 00:00:00 2001 From: ascherbakoff Date: Mon, 29 Oct 2018 16:39:44 +0300 Subject: [PATCH 053/403] IGNITE-8902 Fixed rollback of GridDhtTxRemote for one-phase commit - Fixes #5089. Signed-off-by: Alexey Goncharuk (cherry picked from commit 996a1d5ed8af0034dd1541b206bda0ac9467f981) --- .../cache/GridCacheSharedContext.java | 1 - .../dht/GridDhtTxPrepareFuture.java | 140 +++++------- .../cache/transactions/IgniteTxHandler.java | 1 + .../cache/transactions/IgniteTxManager.java | 3 +- .../TestRecordingCommunicationSpi.java | 20 ++ ...TxRollbackOnTimeoutOnePhaseCommitTest.java | 215 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite6.java | 2 + 7 files changed, 302 insertions(+), 80 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index efbbf072db3fd..94d8c1c6b2aaa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -1149,5 +1149,4 @@ public void readOnlyMode(boolean readOnlyMode) { public void setTxManager(IgniteTxManager txMgr) { this.txMgr = txMgr; } - } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java index 609bff801e406..d10d329fecf51 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; @@ -217,6 +218,9 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture 0 ? new PrepareTimeoutObject(timeout) : null; + + if (tx.onePhaseCommit()) + timeoutAddedLatch = new CountDownLatch(1); } /** {@inheritDoc} */ @@ -692,6 +699,14 @@ private boolean mapIfLocked() { if (!MAPPED_UPD.compareAndSet(this, 0, 1)) return false; + if (timeoutObj != null && tx.onePhaseCommit()) { + U.awaitQuiet(timeoutAddedLatch); + + // Disable timeouts after all locks are acquired for one-phase commit or partition desync will occur. + if (!cctx.time().removeTimeoutObject(timeoutObj)) + return true; // Should not proceed with prepare if tx is already timed out. + } + if (forceKeysFut == null || (forceKeysFut.isDone() && forceKeysFut.error() == null)) prepare0(); else { @@ -730,7 +745,7 @@ private boolean mapIfLocked() { tx.clearPrepareFuture(this); // Do not commit one-phase commit transaction if originating node has near cache enabled. - if (tx.onePhaseCommit() && tx.commitOnPrepare()) { + if (tx.commitOnPrepare()) { assert last; Throwable prepErr = this.err; @@ -740,61 +755,55 @@ private boolean mapIfLocked() { onComplete(res); - if (tx.commitOnPrepare()) { - if (tx.markFinalizing(IgniteInternalTx.FinalizationStatus.USER_FINISH)) { - CIX1> resClo = - new CIX1>() { - @Override public void applyx(IgniteInternalFuture fut) { - if(res.error() == null && fut.error() != null) - res.error(fut.error()); + if (tx.markFinalizing(IgniteInternalTx.FinalizationStatus.USER_FINISH)) { + CIX1> resClo = + new CIX1>() { + @Override public void applyx(IgniteInternalFuture fut) { + if (res.error() == null && fut.error() != null) + res.error(fut.error()); - if (REPLIED_UPD.compareAndSet(GridDhtTxPrepareFuture.this, 0, 1)) - sendPrepareResponse(res); - } - }; - - try { - if (prepErr == null) { - try { - tx.commitAsync().listen(resClo); - } - catch (Throwable e) { - res.error(e); + if (REPLIED_UPD.compareAndSet(GridDhtTxPrepareFuture.this, 0, 1)) + sendPrepareResponse(res); + } + }; - tx.systemInvalidate(true); + try { + if (prepErr == null) { + try { + tx.commitAsync().listen(resClo); + } + catch (Throwable e) { + res.error(e); - try { - tx.rollbackAsync().listen(resClo); - } - catch (Throwable e1) { - e.addSuppressed(e1); - } + tx.systemInvalidate(true); - throw e; - } - } - else if (!cctx.kernalContext().isStopping()) { try { tx.rollbackAsync().listen(resClo); } - catch (Throwable e) { - if (err != null) - err.addSuppressed(e); - - throw err; + catch (Throwable e1) { + e.addSuppressed(e1); } + + throw e; } } - catch (Throwable e){ - tx.logTxFinishErrorSafe(log, true, e); + else if (!cctx.kernalContext().isStopping()) { + try { + tx.rollbackAsync().listen(resClo); + } + catch (Throwable e) { + if (err != null) + err.addSuppressed(e); - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + throw err; + } } } - } - else { - if (REPLIED_UPD.compareAndSet(this, 0, 1)) - sendPrepareResponse(res); + catch (Throwable e) { + tx.logTxFinishErrorSafe(log, true, e); + + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } } return true; @@ -1063,9 +1072,14 @@ public void prepare(GridNearTxPrepareRequest req) { readyLocks(); - if (timeoutObj != null && !isDone()) { - // Start timeout tracking after 'readyLocks' to avoid race with timeout processing. + // Start timeout tracking after 'readyLocks' to avoid race with timeout processing. + if (timeoutObj != null) { cctx.time().addTimeoutObject(timeoutObj); + + // Fix race with add/remove timeout object if locks are mapped from another + // thread before timeout object is enqueued. + if (tx.onePhaseCommit()) + timeoutAddedLatch.countDown(); } mapIfLocked(); @@ -1217,8 +1231,6 @@ private IgniteTxOptimisticCheckedException versionCheckError(IgniteTxEntry entry * */ private void prepare0() { - boolean skipInit = false; - try { if (tx.serializable() && tx.optimistic()) { IgniteCheckedException err0; @@ -1253,8 +1265,6 @@ private void prepare0() { } } - IgniteInternalFuture waitCrdCntrFut = null; - if (req.requestMvccCounter()) { assert last; @@ -1295,37 +1305,11 @@ private void prepare0() { if (isDone()) return; - if (last) { - if (waitCrdCntrFut != null) { - skipInit = true; - - waitCrdCntrFut.listen(new IgniteInClosure>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - fut.get(); - - sendPrepareRequests(); - - markInitialized(); - } - catch (Throwable e) { - U.error(log, "Failed to get mvcc version for tx [txId=" + tx.nearXidVersion() + - ", err=" + e + ']', e); - - GridNearTxPrepareResponse res = createPrepareResponse(e); - - onDone(res, res.error()); - } - } - }); - } - else - sendPrepareRequests(); - } + if (last) + sendPrepareRequests(); } finally { - if (!skipInit) - markInitialized(); + markInitialized(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 75e208719689a..0c51e1502aae9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -1685,6 +1685,7 @@ private void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean committe single, req.storeWriteThrough()); + tx.onePhaseCommit(req.onePhaseCommit()); tx.writeVersion(req.writeVersion()); tx = ctx.tm().onCreated(null, tx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java index 0c2ca347ab885..032df224d4a05 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java @@ -891,7 +891,8 @@ public void prepareTx(IgniteInternalTx tx, @Nullable Collection e throw new IgniteCheckedException("Transaction is marked for rollback: " + tx); } - if (tx.remainingTime() == -1) { + // One-phase commit tx cannot timeout on prepare because it is expected to be committed. + if (tx.remainingTime() == -1 && !tx.onePhaseCommit()) { tx.setRollbackOnly(); throw new IgniteTxTimeoutCheckedException("Transaction timed out: " + this); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index 7b68a6bfd8337..988395f473573 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -217,6 +217,26 @@ public void waitForBlocked(int size) throws InterruptedException { } } + /** + * @param size Size + * @param timeout Timeout. + * @throws InterruptedException + */ + public boolean waitForBlocked(int size, long timeout) throws InterruptedException { + long t0 = U.currentTimeMillis() + timeout; + + synchronized (this) { + while (blockedMsgs.size() < size) { + wait(1000); + + if (U.currentTimeMillis() >= t0) + return false; + } + } + + return true; + } + /** * @throws InterruptedException If interrupted. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java new file mode 100644 index 0000000000000..2375d9ed03231 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.transactions; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse; +import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionTimeoutException; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; +import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests rollback on timeout scenarios for one-phase commit protocol. + */ +public class TxRollbackOnTimeoutOnePhaseCommitTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static final int GRID_CNT = 2; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + + boolean client = igniteInstanceName.startsWith("client"); + + cfg.setClientMode(client); + + if (!client) { + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + ccfg.setAtomicityMode(TRANSACTIONAL); + ccfg.setBackups(1); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + ccfg.setOnheapCacheEnabled(false); + + cfg.setCacheConfiguration(ccfg); + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + startGridsMultiThreaded(GRID_CNT); + + startGrid("client"); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** */ + public void testRollbackOnTimeoutPartitionDesyncPessimistic() throws Exception { + doTestRollbackOnTimeoutPartitionDesync(PESSIMISTIC); + } + + /** */ + public void testRollbackOnTimeoutPartitionDesyncOptimistic() throws Exception { + doTestRollbackOnTimeoutPartitionDesync(OPTIMISTIC); + } + + /** */ + public void testUnlockOptimistic() throws IgniteCheckedException { + IgniteEx client = grid("client"); + + assertNotNull(client.cache(DEFAULT_CACHE_NAME)); + + int key = 0; + + CountDownLatch lock = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + + IgniteInternalFuture fut = runAsync(() -> { + try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) { + client.cache(DEFAULT_CACHE_NAME).put(key, key + 1); + + lock.countDown(); + + try { + assertTrue(U.await(finish, 30, TimeUnit.SECONDS)); + } + catch (IgniteInterruptedCheckedException e) { + fail(); + } + + tx.commit(); + } + }); + + try (Transaction tx = client.transactions().txStart(OPTIMISTIC, REPEATABLE_READ, 200, 1)) { + try { + assertTrue(U.await(lock, 30, TimeUnit.SECONDS)); + } + catch (IgniteInterruptedCheckedException e) { + fail(); + } + + client.cache(DEFAULT_CACHE_NAME).put(key, key); + + tx.commit(); + + // fail(); // TODO IGNITE-10027 throw timeout exception for optimistic timeout. + } + catch (Exception e) { + assertTrue(e.getClass().getName(), X.hasCause(e, TransactionTimeoutException.class)); + } + + assertNull(client.cache(DEFAULT_CACHE_NAME).get(key)); + + finish.countDown(); + + fut.get(); + + assertEquals(1, client.cache(DEFAULT_CACHE_NAME).get(key)); + } + + /** */ + private void doTestRollbackOnTimeoutPartitionDesync(TransactionConcurrency concurrency) throws Exception { + IgniteEx client = grid("client"); + + assertNotNull(client.cache(DEFAULT_CACHE_NAME)); + + int key = 0; + + Ignite primary = primaryNode(key, DEFAULT_CACHE_NAME); + Ignite backup = backupNode(key, DEFAULT_CACHE_NAME); + + TestRecordingCommunicationSpi backupSpi = TestRecordingCommunicationSpi.spi(backup); + backupSpi.blockMessages(GridDhtTxPrepareResponse.class, primary.name()); + + IgniteInternalFuture fut = runAsync(() -> { + try { + backupSpi.waitForBlocked(1, 5000); + } + catch (InterruptedException e) { + fail(); + } + + doSleep(500); + + backupSpi.stopBlock(); + }); + + try (Transaction tx = client.transactions().txStart(concurrency, REPEATABLE_READ, 500, 1)) { + client.cache(DEFAULT_CACHE_NAME).put(key, key); + + tx.commit(); + } + catch (Exception e) { + assertTrue(e.getClass().getName(), X.hasCause(e, TransactionTimeoutException.class)); + } + + fut.get(); + + IdleVerifyResultV2 res = idleVerify(client, DEFAULT_CACHE_NAME); + + if (res.hasConflicts()) { + StringBuilder b = new StringBuilder(); + + res.print(b::append); + + fail(b.toString()); + } + + checkFutures(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index 1269d0d1bbb40..bd7236ebe9f9b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -46,6 +46,7 @@ import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnIncorrectParamsTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest; +import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutOnePhaseCommitTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest; import org.apache.ignite.internal.processors.cache.transactions.TxStateChangeEventTest; @@ -80,6 +81,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(TxRollbackAsyncTest.class); suite.addTestSuite(TxRollbackAsyncNearCacheTest.class); suite.addTestSuite(TxRollbackOnTopologyChangeTest.class); + suite.addTestSuite(TxRollbackOnTimeoutOnePhaseCommitTest.class); suite.addTestSuite(TxOptimisticPrepareOnUnstableTopologyTest.class); From ec0742233faf4e7a3bf0cfdfd8f9c0e0978aefa2 Mon Sep 17 00:00:00 2001 From: Aleksei Scherbakov Date: Fri, 26 Oct 2018 13:00:22 +0300 Subject: [PATCH 054/403] IGNITE-8873 Added methods to preload partitions to page memory - Fixes #5053. Signed-off-by: Alexey Goncharuk (cherry picked from commit 28e3dec5b4c9bffcca5391cac8bee824973fc7a4) --- .../java/org/apache/ignite/IgniteCache.java | 49 +- .../cache/GatewayProtectedCacheProxy.java | 38 +- .../processors/cache/GridCacheAdapter.java | 134 ++++- .../processors/cache/GridCacheProxyImpl.java | 36 ++ .../cache/IgniteCacheOffheapManager.java | 22 +- .../cache/IgniteCacheOffheapManagerImpl.java | 10 + .../cache/IgniteCacheProxyImpl.java | 30 + .../processors/cache/IgniteInternalCache.java | 23 + .../GridDhtPartitionsReservation.java | 2 +- .../cache/local/GridLocalCache.java | 23 + .../persistence/GridCacheOffheapManager.java | 49 ++ .../db/IgnitePdsPartitionPreloadTest.java | 512 ++++++++++++++++++ .../multijvm/IgniteCacheProcessProxy.java | 15 + .../testsuites/IgnitePdsTestSuite4.java | 3 + .../cache/hibernate/HibernateCacheProxy.java | 15 + .../ApiParity/CacheParityTest.cs | 5 +- 16 files changed, 957 insertions(+), 9 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java index 8479420910f16..70ee0d56f5961 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java @@ -1516,7 +1516,7 @@ public IgniteFuture>> invokeAllAsync(Set lostPartitions(); @@ -1531,4 +1531,51 @@ public IgniteFuture>> invokeAllAsync(Set + * This is useful for fast iteration over cache partition data if persistence is enabled and the data is "cold". + *

+ * Preload will reduce available amount of page memory for subsequent operations and may lead to earlier page + * replacement. + *

+ * This method is irrelevant for in-memory caches. Calling this method on an in-memory cache will result in + * exception. + * + * @param partition Partition. + */ + public void preloadPartition(int partition); + + /** + * Efficiently preloads cache partition into page memory. + *

+ * This is useful for fast iteration over cache partition data if persistence is enabled and the data is "cold". + *

+ * Preload will reduce available amount of page memory for subsequent operations and may lead to earlier page + * replacement. + *

+ * This method is irrelevant for in-memory caches. Calling this method on an in-memory cache will result in + * exception. + * + * @param partition Partition. + * @return A future representing pending completion of the partition preloading. + */ + public IgniteFuture preloadPartitionAsync(int partition); + + /** + * Efficiently preloads cache partition into page memory if it exists on the local node. + *

+ * This is useful for fast iteration over cache partition data if persistence is enabled and the data is "cold". + *

+ * Preload will reduce available amount of page memory for subsequent operations and may lead to earlier page + * replacement. + *

+ * This method is irrelevant for in-memory caches. Calling this method on an in-memory cache will result in + * exception. + * + * @param partition Partition. + * @return {@code True} if partition was preloaded, {@code false} if it doesn't belong to local node. + */ + public boolean localPreloadPartition(int partition); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java index c99eb006002e9..8913634a34416 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java @@ -49,8 +49,8 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterGroup; import org.apache.ignite.internal.AsyncSupportAdapter; -import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils; import org.apache.ignite.internal.GridKernalState; +import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.future.IgniteFutureImpl; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -1494,6 +1494,42 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) { } } + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) { + CacheOperationGate opGate = onEnter(); + + try { + delegate.preloadPartition(part); + } + finally { + onLeave(opGate); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture preloadPartitionAsync(int part) { + CacheOperationGate opGate = onEnter(); + + try { + return delegate.preloadPartitionAsync(part); + } + finally { + onLeave(opGate); + } + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int part) { + CacheOperationGate opGate = onEnter(); + + try { + return delegate.localPreloadPartition(part); + } + finally { + onLeave(opGate); + } + } + /** * Safely get CacheGateway. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 3156d6d662d12..a62c64cb7c189 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -87,9 +87,11 @@ import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl; import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; @@ -136,10 +138,13 @@ import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgniteOutClosure; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.lang.IgniteProductVersion; +import org.apache.ignite.lang.IgniteRunnable; import org.apache.ignite.mxbean.CacheMetricsMXBean; import org.apache.ignite.plugin.security.SecurityPermission; import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.resources.JobContextResource; +import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -149,6 +154,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_RETRIES_COUNT; import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST; import static org.apache.ignite.internal.processors.cache.CacheOperationContext.DFLT_ALLOW_ATOMIC_OPS_IN_TX; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; import static org.apache.ignite.internal.processors.dr.GridDrType.DR_LOAD; import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE; import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER; @@ -179,6 +185,9 @@ public abstract class GridCacheAdapter implements IgniteInternalCache> stash = new ThreadLocal>() { @@ -284,6 +293,9 @@ public abstract class GridCacheAdapter implements IgniteInternalCache executeClearTask(@Nullable Set keys return new GridFinishedFuture<>(); } + /** + * @param part Partition id. + * @return Future. + */ + private IgniteInternalFuture executePreloadTask(int part) throws IgniteCheckedException { + ClusterGroup grp = ctx.grid().cluster().forDataNodes(ctx.name()); + + @Nullable ClusterNode targetNode = ctx.affinity().primaryByPartition(part, ctx.topology().readyTopologyVersion()); + + if (targetNode == null || targetNode.version().compareTo(PRELOAD_PARTITION_SINCE) < 0) { + if (!partPreloadBadVerWarned) { + U.warn(log(), "Attempting to execute partition preloading task on outdated or not mapped node " + + "[targetNodeVer=" + (targetNode == null ? "NA" : targetNode.version()) + + ", minSupportedNodeVer=" + PRELOAD_PARTITION_SINCE + ']'); + + partPreloadBadVerWarned = true; + } + + return new GridFinishedFuture<>(); + } + + return ctx.closures().affinityRun(Collections.singleton(name()), part, + new PartitionPreloadJob(ctx.name(), part), grp.nodes(), null); + } + /** * @param keys Keys. * @param readers Readers flag. @@ -4961,6 +4998,55 @@ private void advance() { return new CacheEntryImpl<>((K)key0, (V)val0, entry.version()); } + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) throws IgniteCheckedException { + if (isLocal()) + ctx.offheap().preloadPartition(part); + else + executePreloadTask(part).get(); + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture preloadPartitionAsync(int part) throws IgniteCheckedException { + if (isLocal()) { + return ctx.kernalContext().closure().runLocalSafe(() -> { + try { + ctx.offheap().preloadPartition(part); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + }); + } + else + return executePreloadTask(part); + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException { + if (!ctx.affinityNode()) + return false; + + GridDhtPartitionTopology top = ctx.group().topology(); + + @Nullable GridDhtLocalPartition p = top.localPartition(part, top.readyTopologyVersion(), false); + + if (p == null) + return false; + + try { + if (!p.reserve() || p.state() != OWNING) + return false; + + p.dataStore().preload(); + } + finally { + p.release(); + } + + return true; + } + /** * */ @@ -6691,6 +6777,52 @@ public ClearTask(String cacheName, AffinityTopologyVersion topVer, Set delegate() { } } + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) throws IgniteCheckedException { + CacheOperationContext prev = gate.enter(opCtx); + + try { + delegate.preloadPartition(part); + } + finally { + gate.leave(prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture preloadPartitionAsync(int part) throws IgniteCheckedException { + CacheOperationContext prev = gate.enter(opCtx); + + try { + return delegate.preloadPartitionAsync(part); + } + finally { + gate.leave(prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException { + CacheOperationContext prev = gate.enter(opCtx); + + try { + return delegate.localPreloadPartition(part); + } + finally { + gate.leave(prev); + } + } + /** {@inheritDoc} */ @Override public GridCacheProxyImpl forSubjectId(UUID subjId) { return new GridCacheProxyImpl<>(ctx, delegate, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index e9ec025206d8d..58f05d2e062fa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -23,8 +23,8 @@ import javax.cache.processor.EntryProcessor; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; @@ -589,6 +589,14 @@ public long cacheEntriesCount(int cacheId, boolean primary, boolean backup, Affi */ public long totalPartitionEntriesCount(int part); + /** + * Preload a partition. Must be called under partition reservation for DHT caches. + * + * @param part Partition. + * @throws IgniteCheckedException If failed. + */ + public void preloadPartition(int part) throws IgniteCheckedException; + /** * */ @@ -1054,7 +1062,7 @@ public GridCursor cursor(int cacheId, KeyCacheObject low /** * @param cntr Counter. */ - void updateInitialCounter(long cntr); + public void updateInitialCounter(long cntr); /** * Inject rows cache cleaner. @@ -1068,11 +1076,17 @@ public GridCursor cursor(int cacheId, KeyCacheObject low * * @return PendingTree instance. */ - PendingEntriesTree pendingTree(); + public PendingEntriesTree pendingTree(); /** * Flushes pending update counters closing all possible gaps. */ - void finalizeUpdateCountres(); + public void finalizeUpdateCountres(); + + /** + * Preload a store into page memory. + * @throws IgniteCheckedException If failed. + */ + public void preload() throws IgniteCheckedException; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index e547784fbf40a..c450d0265a437 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -336,6 +336,11 @@ public CacheDataStore dataStore(int part) { } } + /** {@inheritDoc} */ + @Override public void preloadPartition(int p) throws IgniteCheckedException { + throw new IgniteCheckedException("Operation only applicable to caches with enabled persistence"); + } + /** * @param p Partition. * @return Partition data. @@ -2906,6 +2911,11 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw return pendingEntries; } + /** {@inheritDoc} */ + @Override public void preload() throws IgniteCheckedException { + // No-op. + } + /** * @param cctx Cache context. * @param key Key. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java index 776e1cbeb65de..addd189d32c63 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java @@ -1826,6 +1826,36 @@ private void setFuture(IgniteInternalFuture fut) { } } + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) { + try { + delegate.preloadPartition(part); + } + catch (IgniteCheckedException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture preloadPartitionAsync(int part) { + try { + return (IgniteFuture)createFuture(delegate.preloadPartitionAsync(part)); + } + catch (IgniteCheckedException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int part) { + try { + return delegate.localPreloadPartition(part); + } + catch (IgniteCheckedException e) { + throw cacheException(e); + } + } + /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeObject(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java index cba2228f0323e..9d40190434b3c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java @@ -1818,4 +1818,27 @@ public void localLoadCache(@Nullable IgniteBiPredicate p, @Nullable Object * @return A collection of lost partitions if a cache is in recovery state. */ public Collection lostPartitions(); + + /** + * Preload cache partition. + * @param part Partition. + * @throws IgniteCheckedException If failed. + */ + public void preloadPartition(int part) throws IgniteCheckedException; + + /** + * Preload cache partition. + * @param part Partition. + * @return Future to be completed whenever preloading completes. + * @throws IgniteCheckedException If failed. + */ + public IgniteInternalFuture preloadPartitionAsync(int part) throws IgniteCheckedException; + + /** + * Preloads cache partition if it exists on local node. + * @param part Partition. + * @return {@code True} if partition was preloaded, {@code false} if it doesn't belong to local node. + * @throws IgniteCheckedException If failed. + */ + public boolean localPreloadPartition(int part) throws IgniteCheckedException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java index 2682a896e7476..501748628661b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java @@ -179,7 +179,7 @@ public void onPublish(CI1 unpublish) { */ private static void tryEvict(GridDhtLocalPartition[] parts) { if (parts == null) // Can be not initialized yet. - return ; + return; for (GridDhtLocalPartition part : parts) tryEvict(part); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java index 7b7ac66c0cc31..481a6cf3d8a1b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java @@ -232,4 +232,27 @@ else if (modes.heap) @Override public long localSizeLong(int part, CachePeekMode[] peekModes) throws IgniteCheckedException { return localSizeLong(peekModes); } + + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) throws IgniteCheckedException { + ctx.offheap().preloadPartition(part); + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture preloadPartitionAsync(int part) throws IgniteCheckedException { + return ctx.closures().callLocalSafe(new Callable() { + @Override public Void call() throws Exception { + preloadPartition(part); + + return null; + } + }); + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException { + ctx.offheap().preloadPartition(part); + + return true; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 240fbbe8fdde0..5f6511dcfd046 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -38,6 +38,7 @@ import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageSupport; +import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; @@ -869,6 +870,21 @@ private Metas getOrAllocateCacheMetas() throws IgniteCheckedException { return size; } + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) throws IgniteCheckedException { + if (grp.isLocal()) { + dataStore(part).preload(); + + return; + } + + GridDhtLocalPartition locPart = grp.topology().localPartition(part, AffinityTopologyVersion.NONE, false, false); + + assert locPart != null && locPart.reservations() > 0; + + locPart.dataStore().preload(); + } + /** * Calculates free space of all partition data stores - number of bytes available for use in allocated pages. * @@ -1388,6 +1404,31 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException @Override public PendingEntriesTree pendingTree() { return pendingTree0; } + + /** {@inheritDoc} */ + @Override public void preload() throws IgniteCheckedException { + IgnitePageStoreManager pageStoreMgr = ctx.pageStore(); + + if (pageStoreMgr == null) + return; + + final int pages = pageStoreMgr.pages(grp.groupId(), partId); + + long pageId = pageMem.partitionMetaPageId(grp.groupId(), partId); + + // For each page sequentially pin/unpin. + for (int pageNo = 0; pageNo < pages; pageId++, pageNo++) { + long pagePointer = -1; + + try { + pagePointer = pageMem.acquirePage(grp.groupId(), pageId); + } + finally { + if (pagePointer != -1) + pageMem.releasePage(grp.groupId(), pageId, pagePointer); + } + } + } }; pendingTree = pendingTree0; @@ -2245,6 +2286,14 @@ private int purgeExpiredInternal(GridCacheContext cctx, throw new IgniteException(e); } } + + /** {@inheritDoc} */ + @Override public void preload() throws IgniteCheckedException { + CacheDataStore delegate0 = init0(true); + + if (delegate0 != null) + delegate0.preload(); + } } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java new file mode 100644 index 0000000000000..0452ad48874dd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java @@ -0,0 +1,512 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import java.util.Collections; +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; +import javax.cache.Cache; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.LOCAL; + +/** + * Test partition preload for varios cache modes. + */ +public class IgnitePdsPartitionPreloadTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** Test entry count. */ + public static final int ENTRY_CNT = 500; + + /** Grid count. */ + private static final int GRIDS_CNT = 3; + + /** */ + private static final String CLIENT_GRID_NAME = "client"; + + /** */ + public static final String DEFAULT_REGION = "default"; + + /** */ + private Supplier cfgFactory; + + /** */ + private static final String TEST_ATTR = "testId"; + + /** */ + private static final String NO_CACHE_NODE = "node0"; + + /** */ + private static final String PRIMARY_NODE = "node1"; + + /** */ + private static final String BACKUP_NODE = "node2"; + + /** */ + public static final String MEM = "mem"; + + /** */ + public static final int MB = 1024 * 1024; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + cfg.setClientMode(CLIENT_GRID_NAME.equals(gridName)); + + if (!cfg.isClientMode()) { + String val = "node" + getTestIgniteInstanceIndex(gridName); + cfg.setUserAttributes(Collections.singletonMap(TEST_ATTR, val)); + cfg.setConsistentId(val); + } + + DataStorageConfiguration memCfg = new DataStorageConfiguration() + .setDataRegionConfigurations(new DataRegionConfiguration().setName(MEM).setInitialSize(10 * MB)) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration(). + setMetricsEnabled(true). + setMaxSize(50L * MB). + setPersistenceEnabled(true). + setName(DEFAULT_REGION)) + .setWalMode(WALMode.LOG_ONLY) + .setWalSegmentSize(16 * MB) + .setPageSize(1024) + .setMetricsEnabled(true); + + cfg.setDataStorageConfiguration(memCfg); + + cfg.setCacheConfiguration(cfgFactory.get()); + + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + return cfg; + } + + /** + * @param atomicityMode Atomicity mode. + */ + private CacheConfiguration cacheConfiguration(CacheAtomicityMode atomicityMode) { + CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + ccfg.setAffinity(new RendezvousAffinityFunction(false, 32)); + ccfg.setBackups(1); + ccfg.setNodeFilter(new TestIgnitePredicate()); + ccfg.setAtomicityMode(atomicityMode); + + return ccfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** */ + public void testLocalPreloadPartitionClient() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setDataRegionName(MEM); + + startGridsMultiThreaded(GRIDS_CNT); + + IgniteEx client = startGrid("client"); + + assertNotNull(client.cache(DEFAULT_CACHE_NAME)); + + assertFalse(client.cache(DEFAULT_CACHE_NAME).localPreloadPartition(0)); + assertFalse(grid(0).cache(DEFAULT_CACHE_NAME).localPreloadPartition(0)); + } + + /** */ + public void testLocalPreloadPartitionPrimary() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.LOCAL); + } + + /** */ + public void testLocalPreloadPartitionBackup() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition( + () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.LOCAL); + } + + /** */ + public void testPreloadPartitionInMemoryRemote() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setDataRegionName(MEM); + + startGridsMultiThreaded(GRIDS_CNT); + + IgniteEx client = startGrid("client"); + + assertNotNull(client.cache(DEFAULT_CACHE_NAME)); + + try { + client.cache(DEFAULT_CACHE_NAME).preloadPartition(0); + + fail("Exception is expected"); + } + catch (Exception e) { + log.error("Expected", e); + } + } + + /** */ + public void testPreloadPartitionInMemoryLocal() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setDataRegionName(MEM); + + startGridsMultiThreaded(GRIDS_CNT); + + int key = 0; + + Ignite prim = primaryNode(key, DEFAULT_CACHE_NAME); + + int part = prim.affinity(DEFAULT_CACHE_NAME).partition(key); + + try { + prim.cache(DEFAULT_CACHE_NAME).preloadPartition(part); + + fail("Exception is expected"); + } + catch (Exception e) { + log.error("Expected", e); + } + } + + /** */ + public void testPreloadPartitionTransactionalClientSync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition(() -> { + try { + return startGrid(CLIENT_GRID_NAME); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }, PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionTransactionalClientAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition(() -> { + try { + return startGrid(CLIENT_GRID_NAME); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }, PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionTransactionalNodeFilteredSync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition(() -> grid(0), PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionTransactionalNodeFilteredAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition(() -> grid(0), PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionTransactionalPrimarySync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionTransactionalPrimaryAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionTransactionalBackupSync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition( + () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionTransactionalBackupAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL); + + preloadPartition( + () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionAtomicClientSync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition(() -> { + try { + return startGrid(CLIENT_GRID_NAME); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }, PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionAtomicClientAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition(() -> { + try { + return startGrid(CLIENT_GRID_NAME); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }, PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionAtomicNodeFilteredSync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition(() -> grid(0), PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionAtomicNodeFilteredAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition(() -> grid(0), PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionAtomicPrimarySync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionAtomicPrimaryAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC); + } + + /** */ + public void testPreloadPartitionAtomicBackupSync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition( + () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC); + } + + /** */ + public void testPreloadPartitionAtomicBackupAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(ATOMIC); + + preloadPartition( + () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC); + } + + /** */ + public void testPreloadLocalTransactionalSync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setCacheMode(LOCAL); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC); + } + + /** */ + public void testPreloadLocalTransactionalAsync() throws Exception { + cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setCacheMode(LOCAL); + + preloadPartition( + () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC); + } + + /** + * @param execNodeFactory Test node factory. + * @param preloadMode Preload mode. + */ + private void preloadPartition(Supplier execNodeFactory, PreloadMode preloadMode) throws Exception { + Ignite crd = startGridsMultiThreaded(GRIDS_CNT); + + Ignite testNode = grid(1); + + Object consistentId = testNode.cluster().localNode().consistentId(); + + assertEquals(PRIMARY_NODE, testNode.cluster().localNode().consistentId()); + + boolean locCacheMode = testNode.cache(DEFAULT_CACHE_NAME).getConfiguration(CacheConfiguration.class).getCacheMode() == LOCAL; + + Integer key = primaryKey(testNode.cache(DEFAULT_CACHE_NAME)); + + int preloadPart = crd.affinity(DEFAULT_CACHE_NAME).partition(key); + + int cnt = 0; + + try (IgniteDataStreamer streamer = testNode.dataStreamer(DEFAULT_CACHE_NAME)) { + int k = 0; + + while (cnt < ENTRY_CNT) { + if (testNode.affinity(DEFAULT_CACHE_NAME).partition(k) == preloadPart) { + streamer.addData(k, k); + + cnt++; + } + + k++; + } + } + + forceCheckpoint(); + + stopAllGrids(); + + startGridsMultiThreaded(GRIDS_CNT); + + testNode = G.allGrids().stream(). + filter(ignite -> PRIMARY_NODE.equals(ignite.cluster().localNode().consistentId())).findFirst().get(); + + if (!locCacheMode) + assertEquals(testNode, primaryNode(key, DEFAULT_CACHE_NAME)); + + Ignite execNode = execNodeFactory.get(); + + switch (preloadMode) { + case SYNC: + execNode.cache(DEFAULT_CACHE_NAME).preloadPartition(preloadPart); + + if (locCacheMode) { + testNode = G.allGrids().stream().filter(ignite -> + ignite.cluster().localNode().consistentId().equals(consistentId)).findFirst().get(); + } + + break; + case ASYNC: + execNode.cache(DEFAULT_CACHE_NAME).preloadPartitionAsync(preloadPart).get(); + + if (locCacheMode) { + testNode = G.allGrids().stream().filter(ignite -> + ignite.cluster().localNode().consistentId().equals(consistentId)).findFirst().get(); + } + + break; + case LOCAL: + assertTrue(execNode.cache(DEFAULT_CACHE_NAME).localPreloadPartition(preloadPart)); + + testNode = execNode; // For local preloading testNode == execNode + + break; + } + + long c0 = testNode.dataRegionMetrics(DEFAULT_REGION).getPagesRead(); + + // After partition preloading no pages should be read from store. + List> list = U.arrayList(testNode.cache(DEFAULT_CACHE_NAME).localEntries(), 1000); + + assertEquals(ENTRY_CNT, list.size()); + + assertEquals("Read pages count must be same", c0, testNode.dataRegionMetrics(DEFAULT_REGION).getPagesRead()); + } + + /** */ + private static class TestIgnitePredicate implements IgnitePredicate { + /** {@inheritDoc} */ + @Override public boolean apply(ClusterNode node) { + return !NO_CACHE_NODE.equals(node.attribute(TEST_ATTR)); + } + } + + /** */ + private static class PrimaryNodePredicate implements Predicate { + /** */ + private static final PrimaryNodePredicate INSTANCE = new PrimaryNodePredicate(); + + /** {@inheritDoc} */ + @Override public boolean test(Ignite ignite) { + return PRIMARY_NODE.equals(ignite.cluster().localNode().consistentId()); + } + } + + /** */ + private static class BackupNodePredicate implements Predicate { + /** */ + private static final BackupNodePredicate INSTANCE = new BackupNodePredicate(); + + /** {@inheritDoc} */ + @Override public boolean test(Ignite ignite) { + return BACKUP_NODE.equals(ignite.cluster().localNode().consistentId()); + } + } + + /** */ + private enum PreloadMode { + /** Sync. */ SYNC, + /** Async. */ASYNC, + /** Local. */LOCAL; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java index f81d103712c2e..ecf119b098b5f 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java @@ -687,6 +687,21 @@ private IgniteCacheProcessProxy(String name, boolean async, ExpiryPolicy plc, Ig throw new UnsupportedOperationException("Method should be supported."); } + /** {@inheritDoc} */ + @Override public void preloadPartition(int partId) { + throw new UnsupportedOperationException("Method should be supported."); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture preloadPartitionAsync(int partId) { + throw new UnsupportedOperationException("Method should be supported."); + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int partition) { + throw new UnsupportedOperationException("Method should be supported."); + } + /** {@inheritDoc} */ @Override public IgniteCache withAllowAtomicOpsInTx() { return this; diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java index c16463529d55c..64615e20e0633 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPartitionPreloadTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest; import org.apache.ignite.internal.processors.cache.persistence.file.FileDownloaderTest; @@ -44,6 +45,8 @@ public static TestSuite suite() { suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.class); + suite.addTestSuite(IgnitePdsPartitionPreloadTest.class); + return suite; } diff --git a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java index fdb87f0c20201..0b659188941f2 100644 --- a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java +++ b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java @@ -647,6 +647,21 @@ public HibernateKeyTransformer keyTransformer(){ return delegate.lostPartitions(); } + /** {@inheritDoc} */ + @Override public void preloadPartition(int part) throws IgniteCheckedException { + delegate.preloadPartition(part); + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture preloadPartitionAsync(int part) throws IgniteCheckedException { + return delegate.preloadPartitionAsync(part); + } + + /** {@inheritDoc} */ + @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException { + return delegate.localPreloadPartition(part); + } + /** {@inheritDoc} */ @Nullable @Override public EntryProcessorResult invoke( @Nullable AffinityTopologyVersion topVer, diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs index 7548740c04711..d0a103fabd6a8 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs @@ -57,7 +57,10 @@ public class CacheParityTest "sizeLongAsync", // IGNITE-6563 "localSizeLong", // IGNITE-6563 "enableStatistics", // IGNITE-7276 - "clearStatistics" // IGNITE-9017 + "clearStatistics", // IGNITE-9017 + "preloadPartition", // IGNITE-9998 + "preloadPartitionAsync", // IGNITE-9998 + "localPreloadPartition", // IGNITE-9998 }; ///

From 3e3447b76a466cfc98fff58bb8561687fca138f9 Mon Sep 17 00:00:00 2001 From: sboikov Date: Sat, 6 Oct 2018 12:30:49 +0300 Subject: [PATCH 055/403] IGNITE-8714 Keep old semantic for REMOVED event for internal listeners (cherry picked from commit 6e0ff06f8e309657a16c94da605348d9c3b804ad) --- .../cache/distributed/dht/atomic/GridDhtAtomicCache.java | 3 ++- .../cache/query/continuous/CacheContinuousQueryManager.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 8edefa23188e4..c7b6677ebdbdc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -1793,7 +1793,8 @@ private void updateAllAsyncInternal0( // This call will convert entry processor invocation results to cache object instances. // Must be done outside topology read lock to avoid deadlocks. - res.returnValue().marshalResult(ctx); + if (res.returnValue() != null) + res.returnValue().marshalResult(ctx); break; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java index 4d4be7e560575..c41e1a3108508 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java @@ -406,7 +406,7 @@ public void onEntryUpdated( cctx.cacheId(), evtType, key, - evtType == REMOVED && lsnr.oldValueRequired() ? oldVal : newVal, + (!internal && evtType == REMOVED && lsnr.oldValueRequired()) ? oldVal : newVal, lsnr.oldValueRequired() ? oldVal : null, lsnr.keepBinary(), partId, From 3f38713ef69968093bdc5a41b3c0e81e129490fc Mon Sep 17 00:00:00 2001 From: a-polyakov Date: Tue, 30 Oct 2018 18:07:53 +0300 Subject: [PATCH 056/403] IGNITE-9990 Control.sh utility should request a password if necessary - Fixes #5079. Signed-off-by: Alexey Goncharuk --- .../internal/commandline/Arguments.java | 26 +++- .../internal/commandline/CommandHandler.java | 114 ++++++++++++------ .../CommandHandlerParsingTest.java | 22 +--- 3 files changed, 102 insertions(+), 60 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java index 5b8a0dcd6c005..6367eef516101 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java @@ -71,10 +71,10 @@ public class Arguments { */ private String walArgs; - /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/ + /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}. */ private long pingTimeout; - /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/ + /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */ private long pingInterval; /** @@ -94,8 +94,8 @@ public class Arguments { * @param autoConfirmation Auto confirmation flag. */ public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct, - String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs, - Long pingTimeout, Long pingInterval, boolean autoConfirmation) { + String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs, + Long pingTimeout, Long pingInterval, boolean autoConfirmation) { this.cmd = cmd; this.host = host; this.port = port; @@ -136,17 +136,31 @@ public String port() { /** * @return user name */ - public String user() { + public String getUserName() { return user; } + /** + * @param user New user name. + */ + public void setUserName(String user) { + this.user = user; + } + /** * @return password */ - public String password() { + public String getPassword() { return pwd; } + /** + * @param pwd New password. + */ + public void setPassword(String pwd) { + this.pwd = pwd; + } + /** * @return baseline action */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index c1853f8531df3..167a806f777e2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.commandline; +import java.io.Console; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -117,6 +118,7 @@ import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.plugin.security.SecurityCredentials; import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider; +import org.apache.ignite.plugin.security.SecurityCredentialsProvider; import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND; import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR; @@ -1855,12 +1857,6 @@ Arguments parseAndValidate(List rawArgs) { Command cmd = commands.get(0); - boolean hasUsr = F.isEmpty(user); - boolean hasPwd = F.isEmpty(pwd); - - if (hasUsr != hasPwd) - throw new IllegalArgumentException("Both user and password should be specified"); - return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs, pingTimeout, pingInterval, autoConfirmation); } @@ -2421,50 +2417,100 @@ public int execute(List rawArgs) { clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port())); - if (!F.isEmpty(args.user())) { - clientCfg.setSecurityCredentialsProvider( - new SecurityCredentialsBasicProvider(new SecurityCredentials(args.user(), args.password()))); - } + boolean tryConnectAgain = true; - try (GridClient client = GridClientFactory.start(clientCfg)) { - switch (args.command()) { - case ACTIVATE: - activate(client); + int tryConnectMaxCount=3; - break; + while (tryConnectAgain) { + tryConnectAgain = false; - case DEACTIVATE: - deactivate(client); + if (!F.isEmpty(args.getUserName())) { + SecurityCredentialsProvider securityCredential = clientCfg.getSecurityCredentialsProvider(); - break; + if (securityCredential == null) { + securityCredential = new SecurityCredentialsBasicProvider( + new SecurityCredentials(args.getUserName(), args.getPassword()) + ); - case STATE: - state(client); + clientCfg.setSecurityCredentialsProvider(securityCredential); + } + final SecurityCredentials credential = securityCredential.credentials(); + credential.setLogin(args.getUserName()); + credential.setPassword(args.getPassword()); + } - break; + try (GridClient client = GridClientFactory.start(clientCfg)) { + switch (args.command()) { + case ACTIVATE: + activate(client); - case BASELINE: - baseline(client, args.baselineAction(), args.baselineArguments()); + break; - break; + case DEACTIVATE: + deactivate(client); - case TX: - transactions(client, args.transactionArguments()); + break; - break; + case STATE: + state(client); - case CACHE: - cache(client, args.cacheArgs()); + break; - break; + case BASELINE: + baseline(client, args.baselineAction(), args.baselineArguments()); - case WAL: - wal(client, args.walAction(), args.walArguments()); + break; - break; + case TX: + transactions(client, args.transactionArguments()); + + break; + + case CACHE: + cache(client, args.cacheArgs()); + + break; + + case WAL: + wal(client, args.walAction(), args.walArguments()); + + break; + } } - } + catch (Throwable e) { + if (tryConnectMaxCount > 0 && isAuthError(e)) { + System.out.println("Authentication error, try connection again."); + + final Console console = System.console(); + + if (console != null) { + if (F.isEmpty(args.getUserName())) + args.setUserName(console.readLine("user: ")); + + args.setPassword(new String(console.readPassword("password: "))); + } + else { + Scanner scanner = new Scanner(System.in); + + if (F.isEmpty(args.getUserName())){ + System.out.println("user: "); + + args.setUserName(scanner.next()); + } + + System.out.println("password: "); + + args.setPassword(scanner.next()); + } + + tryConnectAgain = true; + tryConnectMaxCount--; + } + else + throw e; + } + } return EXIT_CODE_OK; } catch (IllegalArgumentException e) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 0ac5d1a4483b0..988b1d7e94a40 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -194,28 +194,10 @@ public void testParseAndValidateUserAndPassword() { e.printStackTrace(); } - try { - hnd.parseAndValidate(asList("--user", "testUser", cmd.text())); - - fail("expected exception: Both user and password should be specified"); - } - catch (IllegalArgumentException e) { - e.printStackTrace(); - } - - try { - hnd.parseAndValidate(asList("--password", "testPass", cmd.text())); - - fail("expected exception: Both user and password should be specified"); - } - catch (IllegalArgumentException e) { - e.printStackTrace(); - } - Arguments args = hnd.parseAndValidate(asList("--user", "testUser", "--password", "testPass", cmd.text())); - assertEquals("testUser", args.user()); - assertEquals("testPass", args.password()); + assertEquals("testUser", args.getUserName()); + assertEquals("testPass", args.getPassword()); assertEquals(cmd, args.command()); } } From f18b3d7757ce658e39c2a2fe64eef31153c1fe89 Mon Sep 17 00:00:00 2001 From: Denis Mekhanikov Date: Tue, 30 Oct 2018 19:08:58 +0300 Subject: [PATCH 057/403] IGNITE-9009 Do not notify local CQ listeners on partition reassignment - Fixes #4403. Signed-off-by: Alexey Goncharuk (cherry picked from commit 0193012b27e73f269b33e10258eac7df89d54b68) --- .../ignite/cache/query/ContinuousQuery.java | 11 +- .../CacheContinuousQueryHandler.java | 16 +- .../CacheContinuousQueryListener.java | 8 +- .../CacheContinuousQueryManager.java | 10 +- .../ContinuousQueryReassignmentTest.java | 170 ++++++++++++++++++ .../IgniteCacheQuerySelfTestSuite3.java | 2 + 6 files changed, 205 insertions(+), 12 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryReassignmentTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/ContinuousQuery.java b/modules/core/src/main/java/org/apache/ignite/cache/query/ContinuousQuery.java index e4d6d0ad36837..0d1444b7eac5c 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/query/ContinuousQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/ContinuousQuery.java @@ -213,7 +213,16 @@ public CacheEntryEventSerializableFilter getRemoteFilter() { return (ContinuousQuery)super.setPageSize(pageSize); } - /** {@inheritDoc} */ + /** + * Sets whether this query should be executed on local node only. + * + * Note: backup event queues are not kept for local continuous queries. It may lead to loss of notifications in case + * of node failures. Use {@link ContinuousQuery#setRemoteFilterFactory(Factory)} to register cache event listeners + * on all cache nodes, if delivery guarantee is required. + * + * @param loc Local flag. + * @return {@code this} for chaining. + */ @Override public ContinuousQuery setLocal(boolean loc) { return (ContinuousQuery)super.setLocal(loc); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java index d1640c64e82d1..ade360a1d60d3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java @@ -144,7 +144,7 @@ public class CacheContinuousQueryHandler implements GridContinuousHandler private transient boolean skipPrimaryCheck; /** */ - private boolean locCache; + private transient boolean locOnly; /** */ private boolean keepBinary; @@ -247,10 +247,10 @@ public void notifyExisting(boolean notifyExisting) { } /** - * @param locCache Local cache. + * @param locOnly Local only. */ - public void localCache(boolean locCache) { - this.locCache = locCache; + public void localOnly(boolean locOnly) { + this.locOnly = locOnly; } /** @@ -514,7 +514,7 @@ public void keepBinary(boolean keepBinary) { skipCtx = new CounterSkipContext(part, cntr, topVer); if (loc) { - assert !locCache; + assert !locOnly; final Collection> evts = handleEvent(ctx, skipCtx.entry()); @@ -583,6 +583,10 @@ public void keepBinary(boolean keepBinary) { private String taskName() { return ctx.security().enabled() ? ctx.task().resolveTaskName(taskHash) : null; } + + @Override public boolean isPrimaryOnly() { + return locOnly && !skipPrimaryCheck; + } }; CacheContinuousQueryManager mgr = manager(ctx); @@ -860,7 +864,7 @@ private void onEntryUpdate(CacheContinuousQueryEvent evt, boolean notify, boolea IgniteClosure, ?> trans = getTransformer(); if (loc) { - if (!locCache) { + if (!locOnly) { Collection> evts = handleEvent(ctx, entry); notifyLocalListener(evts, trans); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java index 7da657fe399dd..029b6d8dca73b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.query.continuous; import java.util.Map; +import org.apache.ignite.cache.query.ContinuousQuery; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; @@ -114,4 +115,9 @@ public void onEntryUpdated(CacheContinuousQueryEvent evt, boolean primary, * @return Whether to notify on existing entries. */ public boolean notifyExisting(); -} \ No newline at end of file + + /** + * @return {@code True} if this listener should be called on events on primary partitions only. + */ + public boolean isPrimaryOnly(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java index c41e1a3108508..6bd3fc2cc549f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java @@ -385,7 +385,7 @@ public void onEntryUpdated( boolean recordIgniteEvt = primary && !internal && cctx.events().isRecordable(EVT_CACHE_QUERY_OBJECT_READ); for (CacheContinuousQueryListener lsnr : lsnrCol.values()) { - if (preload && !lsnr.notifyExisting()) + if (preload && !lsnr.notifyExisting() || lsnr.isPrimaryOnly() && !primary) continue; if (!initialized) { @@ -722,12 +722,14 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr, final CacheContinuousQueryHandler hnd = clsr.apply(); + boolean locOnly = cctx.isLocal() || loc; + hnd.taskNameHash(taskNameHash); hnd.skipPrimaryCheck(skipPrimaryCheck); hnd.notifyExisting(notifyExisting); hnd.internal(internal); hnd.keepBinary(keepBinary); - hnd.localCache(cctx.isLocal()); + hnd.localOnly(locOnly); IgnitePredicate pred = (loc || cctx.config().getCacheMode() == CacheMode.LOCAL) ? F.nodeForNodeId(cctx.localNodeId()) : cctx.group().nodeFilter(); @@ -739,13 +741,13 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr, try { id = cctx.kernalContext().continuous().startRoutine( hnd, - internal && loc, + locOnly, bufSize, timeInterval, autoUnsubscribe, pred).get(); - if (hnd.isQuery() && cctx.userCache() && !onStart) + if (hnd.isQuery() && cctx.userCache() && !locOnly && !onStart) hnd.waitTopologyFuture(cctx.kernalContext()); } catch (NodeStoppingException e) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryReassignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryReassignmentTest.java new file mode 100644 index 0000000000000..2a537cab4d7a4 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryReassignmentTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.query.continuous; + +import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.configuration.FactoryBuilder; +import javax.cache.event.CacheEntryEvent; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheEntryEventSerializableFilter; +import org.apache.ignite.cache.query.ContinuousQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * + */ +public class ContinuousQueryReassignmentTest extends GridCommonAbstractTest { + /** Ip finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration igniteCfg = super.getConfiguration(igniteInstanceName); + + ((TcpDiscoverySpi)igniteCfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + return igniteCfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override public boolean isDebug() { + return true; + } + + /** + * + * @throws Exception If failed. + */ + public void testContinuousQueryNotCalledOnReassignment() throws Exception { + testContinuousQueryNotCalledOnReassignment(false); + } + + /** + * @throws Exception if failed. + */ + public void testLocalContinuousQueryNotCalledOnReassignment() throws Exception { + testContinuousQueryNotCalledOnReassignment(true); + } + + /** + * @param loc If {@code true}, then local continuous query will be tested. + * @throws Exception If failed. + */ + private void testContinuousQueryNotCalledOnReassignment(boolean loc) throws Exception { + Ignite lsnrNode = startGrid(1); + Ignite victim = startGrid(2); + + awaitPartitionMapExchange(); + + CacheConfiguration cacheCfg = new CacheConfiguration<>("cache"); + cacheCfg.setBackups(1); + IgniteCache cache = lsnrNode.getOrCreateCache(cacheCfg); + + AtomicInteger updCntr = new AtomicInteger(); + + listenToUpdates(cache, loc, updCntr, null); + + // Subscribe on all nodes to receive all updates. + if (loc) + listenToUpdates(victim.cache("cache"), true, updCntr, null); + + int updates = 1000; + + for (int i = 0; i < updates; i++) + cache.put(i, Integer.toString(i)); + + assertTrue( + "Failed to wait for continuous query updates. Exp: " + updates + "; actual: " + updCntr.get(), + waitForCondition(() -> updCntr.get() == updates, 10000)); + + victim.close(); + + assertFalse("Continuous query is called on reassignment.", + waitForCondition(() -> updCntr.get() > updates, 2000)); + } + + /** + * @throws Exception If failed. + */ + public void testContinuousQueryWithRemoteFilterNotCalledOnReassignment() throws Exception { + Ignite lsnrNode = startGrid(1); + Ignite victim = startGrid(2); + + awaitPartitionMapExchange(); + + CacheConfiguration cacheCfg = new CacheConfiguration<>("cache"); + cacheCfg.setBackups(1); + IgniteCache cache = lsnrNode.getOrCreateCache(cacheCfg); + + AtomicInteger updCntr = new AtomicInteger(); + + CacheEntryEventSerializableFilter filter = (e) -> e.getKey() % 2 == 0; + + listenToUpdates(cache, false, updCntr, filter); + + int updates = 1000; + + for (int i = 0; i < updates; i++) + cache.put(i, Integer.toString(i)); + + assertTrue( + "Failed to wait for continuous query updates. Exp: " + updates + "; actual: " + updCntr.get(), + waitForCondition(() -> updCntr.get() == updates / 2, 10000)); + + victim.close(); + + assertFalse("Continuous query is called on reassignment.", + waitForCondition(() -> updCntr.get() > updates / 2, 2000)); + } + + /** + * Register a continuous query, that counts updates on the provided cache. + * + * @param cache Cache. + * @param loc If {@code true}, then local continuous query will be registered. + * @param updCntr Update counter. + * @param rmtFilter Remote filter. + */ + private void listenToUpdates(IgniteCache cache, boolean loc, AtomicInteger updCntr, + CacheEntryEventSerializableFilter rmtFilter) { + + ContinuousQuery cq = new ContinuousQuery<>(); + cq.setLocal(loc); + cq.setLocalListener((evts) -> { + for (CacheEntryEvent e : evts) + updCntr.incrementAndGet(); + }); + if (rmtFilter != null) + cq.setRemoteFilterFactory(FactoryBuilder.factoryOf(rmtFilter)); + + cache.query(cq); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java index dbcea2b1ce586..5dde4ccbb22c1 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java @@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.query.continuous.ClientReconnectContinuousQueryTest; import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryMarshallerTest; import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryPeerClassLoadingTest; +import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryReassignmentTest; import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryRemoteFilterMissingInClassPathSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicNearEnabledSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicP2PDisabledSelfTest; @@ -131,6 +132,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(ClientReconnectContinuousQueryTest.class); suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class); suite.addTestSuite(ContinuousQueryMarshallerTest.class); + suite.addTestSuite(ContinuousQueryReassignmentTest.class); suite.addTestSuite(CacheContinuousQueryConcurrentPartitionUpdateTest.class); suite.addTestSuite(CacheContinuousQueryEventBufferTest.class); From de497d2f6824cb2c7d5e950aa77f0c0bd10a10d9 Mon Sep 17 00:00:00 2001 From: vd-pyatkov Date: Wed, 31 Oct 2018 11:54:36 +0300 Subject: [PATCH 058/403] IGNITE-10028 Fixed incorrect handling of page during replacement - Fixes #5093. Signed-off-by: Alexey Goncharuk --- .../GridCacheDatabaseSharedManager.java | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 3843ce1fe86e4..ed44f56ec1582 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -1062,13 +1062,11 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi (fullId, pageBuf, tag) -> { memMetrics.onPageWritten(); - // First of all, write page to disk. - storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag); - - pageBuf.rewind(); + // We can write only page from disk into snapshot. + snapshotMgr.beforePageWrite(fullId); - // Only after write we can write page into snapshot. - snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag); + // Write page to disk. + storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag); AtomicInteger cntr = evictedPagesCntr; @@ -4039,8 +4037,6 @@ private WriteCheckpointPages( private List writePages(Collection writePageIds) throws IgniteCheckedException { ByteBuffer tmpWriteBuf = threadBuf.get(); - long writeAddr = GridUnsafe.bufferAddress(tmpWriteBuf); - List pagesToRetry = new ArrayList<>(); for (FullPageId fullId : writePageIds) { @@ -4095,19 +4091,7 @@ else if (grpId == TxLog.TX_LOG_CACHE_ID) tracker.onDataPageWritten(); } - if (!skipCrc) { - PageIO.setCrc(writeAddr, FastCrc.calcCrc(tmpWriteBuf, pageSize())); - - tmpWriteBuf.rewind(); - } - - int curWrittenPages = writtenPagesCntr.incrementAndGet(); - - snapshotMgr.onPageWrite(fullId, tmpWriteBuf, curWrittenPages, totalPagesToWrite); - - tmpWriteBuf.rewind(); - - PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, false); + PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, true); updStores.computeIfAbsent(store, k -> new LongAdder()).increment(); } From e519bdbb75d7c8f3ca33f07a201e12d9918d7500 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 31 Oct 2018 17:35:25 +0700 Subject: [PATCH 059/403] IGNITE-10036 Visor CMD: Moved "node_startup_by_ssh.sample.ini" to "/config/visor-cmd". (cherry picked from commit 0e7dc7196f4fe7f150e9c633fbe73b3dd8765a16) --- .../visor-cmd}/node_startup_by_ssh.sample.ini | 5 +++++ 1 file changed, 5 insertions(+) rename {bin/include/visorcmd => config/visor-cmd}/node_startup_by_ssh.sample.ini (85%) diff --git a/bin/include/visorcmd/node_startup_by_ssh.sample.ini b/config/visor-cmd/node_startup_by_ssh.sample.ini similarity index 85% rename from bin/include/visorcmd/node_startup_by_ssh.sample.ini rename to config/visor-cmd/node_startup_by_ssh.sample.ini index e50ff29930459..f1d8e01b7b95e 100644 --- a/bin/include/visorcmd/node_startup_by_ssh.sample.ini +++ b/config/visor-cmd/node_startup_by_ssh.sample.ini @@ -13,6 +13,11 @@ ;See the License for the specific language governing permissions and ;limitations under the License. +# ================================================================== +# This is a sample file for Visor CMD to use with "start" command. +# More info: https://apacheignite-tools.readme.io/docs/start-command +# ================================================================== + # Section with settings for host1: [host1] # IP address or host name: From df7c6116775dbd55e65a86affe63603fe0e3a918 Mon Sep 17 00:00:00 2001 From: Evgeny Stanilovskiy Date: Thu, 1 Nov 2018 10:11:44 +0300 Subject: [PATCH 060/403] IGNITE-9418 Avoid initialize file page store manager for caches during PME synchronously - Fixes #4827. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit 31bf38d37ff0b73b24a8fa0034630d833eee1f75) --- .../processors/cache/GridCacheProcessor.java | 2 - .../file/FilePageStoreManager.java | 57 +++++++++++++++---- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 4f8301e405f58..1a169130b7736 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2211,8 +2211,6 @@ private GridCacheContext prepareCacheContext( boolean affNode = checkForAffinityNode(desc, reqNearCfg, ccfg); - preparePageStore(desc, affNode); - CacheGroupContext grp = prepareCacheGroup(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName()); GridCacheContext cacheCtx = createCache(ccfg, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index c6cd9e502c2c2..b6cb163e738bd 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -38,6 +38,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -60,6 +61,7 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.marshaller.Marshaller; @@ -379,16 +381,15 @@ public FilePageStoreManager(GridKernalContext ctx) { CacheStoreHolder old = idxCacheStores.remove(grp.groupId()); - assert old != null : "Missing cache store holder [cache=" + grp.cacheOrGroupName() + - ", locNodeId=" + cctx.localNodeId() + ", gridName=" + cctx.igniteInstanceName() + ']'; + if (old != null) { + IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null); - IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null); + if (destroy) + removeCacheGroupConfigurationData(grp); - if (destroy) - removeCacheGroupConfigurationData(grp); - - if (ex != null) - throw ex; + if (ex != null) + throw ex; + } } /** {@inheritDoc} */ @@ -600,8 +601,9 @@ private CacheStoreHolder initDir(File cacheWorkDir, return new CacheStoreHolder(idxStore, partStores); } - catch (StorageException e) { - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + catch (IgniteCheckedException e) { + if (X.hasCause(e, StorageException.class, IOException.class)) + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); throw e; } @@ -989,6 +991,39 @@ private IgniteCheckedException shutdown(PageStore store, boolean cleanFile, Igni return aggr; } + /** + * Return cache store holedr. + * + * @param grpId Cache group ID. + * @return Cache store holder. + */ + private CacheStoreHolder getHolder(int grpId) throws IgniteCheckedException { + try { + return idxCacheStores.computeIfAbsent(grpId, (key) -> { + CacheGroupDescriptor gDesc = cctx.cache().cacheGroupDescriptors().get(grpId); + + CacheStoreHolder holder0 = null; + + if (gDesc != null) { + if (CU.isPersistentCache(gDesc.config(), cctx.gridConfig().getDataStorageConfiguration())) { + try { + holder0 = initForCache(gDesc, gDesc.config()); + } catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + } + + return holder0; + }); + } catch (IgniteException ex) { + if (X.hasCause(ex, IgniteCheckedException.class)) + throw ex.getCause(IgniteCheckedException.class); + else + throw ex; + } + } + /** * @param grpId Cache group ID. * @param partId Partition ID. @@ -998,7 +1033,7 @@ private IgniteCheckedException shutdown(PageStore store, boolean cleanFile, Igni * Note: visible for testing. */ public PageStore getStore(int grpId, int partId) throws IgniteCheckedException { - CacheStoreHolder holder = idxCacheStores.get(grpId); + CacheStoreHolder holder = getHolder(grpId); if (holder == null) throw new IgniteCheckedException("Failed to get page store for the given cache ID " + From 2352b4a87a39183d1c8ce5a058157ad355fb74cc Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 1 Nov 2018 12:14:19 +0300 Subject: [PATCH 061/403] IGNITE-9988: Enhancements in thin client handlers. This closes #5075. --- ...ientListenerAbstractConnectionContext.java | 6 +-- .../odbc/ClientListenerConnectionContext.java | 6 +++ .../odbc/ClientListenerNioListener.java | 21 +++++++- .../odbc/jdbc/JdbcRequestHandler.java | 49 ++++++++----------- .../processors/odbc/odbc/OdbcUtils.java | 1 + .../platform/client/ClientRequest.java | 8 +-- .../platform/client/ClientRequestHandler.java | 18 +++---- .../security/SecurityContextHolder.java | 15 ++++-- .../query/h2/ddl/DdlStatementsProcessor.java | 4 +- 9 files changed, 73 insertions(+), 55 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java index 1c19d5599cd20..856868de6ffc8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java @@ -65,10 +65,8 @@ public GridKernalContext kernalContext() { return ctx; } - /** - * @return Security context. - */ - @Nullable public SecurityContext securityContext() { + /** {@inheritDoc} */ + @Nullable @Override public SecurityContext securityContext() { return secCtx; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerConnectionContext.java index b693cb6f6ef38..c39bfe20a443a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerConnectionContext.java @@ -20,6 +20,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.binary.BinaryReaderExImpl; import org.apache.ignite.internal.processors.authentication.AuthorizationContext; +import org.apache.ignite.internal.processors.security.SecurityContext; import org.jetbrains.annotations.Nullable; /** @@ -76,4 +77,9 @@ void initializeFromHandshake(ClientListenerProtocolVersion ver, BinaryReaderExIm * @return authorization context. */ @Nullable AuthorizationContext authorizationContext(); + + /** + * @return Security context. + */ + @Nullable SecurityContext securityContext(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java index 0eb6ac4d480e3..debef427d3f8b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java @@ -27,11 +27,14 @@ import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream; import org.apache.ignite.internal.binary.streams.BinaryInputStream; +import org.apache.ignite.internal.processors.authentication.AuthorizationContext; import org.apache.ignite.internal.processors.authentication.IgniteAccessControlException; import org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext; import org.apache.ignite.internal.processors.odbc.odbc.OdbcConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientStatus; +import org.apache.ignite.internal.processors.security.SecurityContext; +import org.apache.ignite.internal.processors.security.SecurityContextHolder; import org.apache.ignite.internal.util.GridSpinBusyLock; import org.apache.ignite.internal.util.nio.GridNioServerListenerAdapter; import org.apache.ignite.internal.util.nio.GridNioSession; @@ -159,7 +162,23 @@ public ClientListenerNioListener(GridKernalContext ctx, GridSpinBusyLock busyLoc ses.remoteAddress() + ", req=" + req + ']'); } - ClientListenerResponse resp = handler.handle(req); + ClientListenerResponse resp; + + AuthorizationContext authCtx = connCtx.authorizationContext(); + SecurityContext oldSecCtx = SecurityContextHolder.push(connCtx.securityContext()); + + if (authCtx != null) + AuthorizationContext.context(authCtx); + + try { + resp = handler.handle(req); + } + finally { + SecurityContextHolder.pop(oldSecCtx); + + if (authCtx != null) + AuthorizationContext.clear(); + } if (resp != null) { if (log.isDebugEnabled()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java index d5a277e5a391a..e58b7efab214c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java @@ -404,41 +404,32 @@ private ClientListenerResponse processBulkLoadFileBatch(JdbcBulkLoadBatchRequest * or due to {@code IOException} during network operations. */ public void onDisconnect() { - if (busyLock.enterBusy()) - { - if (worker != null) { - worker.cancel(); + if (worker != null) { + worker.cancel(); - try { - worker.join(); - } - catch (InterruptedException e) { - // No-op. - } + try { + worker.join(); } + catch (InterruptedException e) { + // No-op. + } + } - try - { - for (JdbcQueryCursor cursor : qryCursors.values()) - cursor.close(); - - for (JdbcBulkLoadProcessor processor : bulkLoadRequests.values()) { - try { - processor.close(); - } - catch (Exception e) { - U.error(null, "Error closing JDBC bulk load processor.", e); - } - } - - bulkLoadRequests.clear(); + for (JdbcQueryCursor cursor : qryCursors.values()) + cursor.close(); - U.close(cliCtx, log); + for (JdbcBulkLoadProcessor processor : bulkLoadRequests.values()) { + try { + processor.close(); } - finally { - busyLock.leaveBusy(); + catch (Exception e) { + U.error(null, "Error closing JDBC bulk load processor.", e); } } + + bulkLoadRequests.clear(); + + U.close(cliCtx, log); } /** @@ -1082,7 +1073,7 @@ private JdbcResponse exceptionToResult(Exception e) { if (e instanceof IgniteSQLException) return new JdbcResponse(((IgniteSQLException) e).statusCode(), e.getMessage()); else - return new JdbcResponse(IgniteQueryErrorCode.UNKNOWN, e.toString()); + return new JdbcResponse(IgniteQueryErrorCode.UNKNOWN, e.getMessage()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java index a1c67aad62f3d..966d7bbc4d9ce 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java @@ -173,6 +173,7 @@ public static String tryRetrieveH2ErrorMessage(Throwable err) { String msg = err.getMessage(); Throwable e = err.getCause(); + while (e != null) { if (e.getClass().getCanonicalName().equals("org.h2.jdbc.JdbcSQLException")) { msg = e.getMessage(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java index 799b3e733f410..a4000efc009da 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java @@ -82,9 +82,9 @@ protected static void runWithSecurityExceptionHandler(Runnable runnable) { * Authorize for specified permission. */ protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) { - SecurityContext secCtx = ctx.securityContext(); - - if (secCtx != null) - runWithSecurityExceptionHandler(() -> ctx.kernalContext().security().authorize(null, perm, secCtx)); +// SecurityContext secCtx = ctx.securityContext(); +// +// if (secCtx != null) +// runWithSecurityExceptionHandler(() -> ctx.kernalContext().security().authorize(null, perm, secCtx)); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java index 5ed0d38d8eac5..8fe4e5df12df7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java @@ -22,7 +22,7 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerRequest; import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler; import org.apache.ignite.internal.processors.odbc.ClientListenerResponse; -import org.apache.ignite.internal.processors.security.SecurityContextHolder; +import org.apache.ignite.plugin.security.SecurityException; /** * Thin client request handler. @@ -48,19 +48,15 @@ public class ClientRequestHandler implements ClientListenerRequestHandler { /** {@inheritDoc} */ @Override public ClientListenerResponse handle(ClientListenerRequest req) { - if (authCtx != null) { - AuthorizationContext.context(authCtx); - SecurityContextHolder.set(ctx.securityContext()); - } - try { return ((ClientRequest)req).process(ctx); } - finally { - if (authCtx != null) - AuthorizationContext.clear(); - - SecurityContextHolder.clear(); + catch (SecurityException ex) { + throw new IgniteClientException( + ClientStatus.SECURITY_VIOLATION, + "Client is not authorized to perform this operation", + ex + ); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java index 14d70c97cbfdb..d01071166ea73 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java @@ -39,15 +39,22 @@ public class SecurityContextHolder { * Set security context. * * @param ctx Context. + * @return Old context. */ - public static void set(@Nullable SecurityContext ctx) { + public static SecurityContext push(@Nullable SecurityContext ctx) { + SecurityContext oldCtx = CTX.get(); + CTX.set(ctx); + + return oldCtx; } /** - * Clear security context. + * Pop security context. + * + * @param oldCtx Old context. */ - public static void clear() { - set(null); + public static void pop(@Nullable SecurityContext oldCtx) { + CTX.set(oldCtx); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java index 5c2865abe1350..94e39ef2870c0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java @@ -328,7 +328,7 @@ else if (stmt0 instanceof GridSqlDropIndex) { } } else if (stmt0 instanceof GridSqlCreateTable) { - ctx.security().authorize(null, SecurityPermission.CACHE_CREATE, SecurityContextHolder.get()); + ctx.security().authorize(null, SecurityPermission.CACHE_CREATE, null); GridSqlCreateTable cmd = (GridSqlCreateTable)stmt0; @@ -361,7 +361,7 @@ else if (stmt0 instanceof GridSqlCreateTable) { } } else if (stmt0 instanceof GridSqlDropTable) { - ctx.security().authorize(null, SecurityPermission.CACHE_DESTROY, SecurityContextHolder.get()); + ctx.security().authorize(null, SecurityPermission.CACHE_DESTROY, null); GridSqlDropTable cmd = (GridSqlDropTable)stmt0; From f91069d33d4ddcebec40113d78c21511435f2864 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Thu, 1 Nov 2018 16:25:10 +0700 Subject: [PATCH 062/403] IGNITE-7926 Web Console agent: Fixed incorrect classpath - Fixes #5221. (cherry picked from commit 6f9c702cd764963208c8962d7ef35b2c340dbc08) --- .../web-agent/bin/ignite-web-agent.sh | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/modules/web-console/web-agent/bin/ignite-web-agent.sh b/modules/web-console/web-agent/bin/ignite-web-agent.sh index 3a649e3aaae09..74a8bf5d54212 100755 --- a/modules/web-console/web-agent/bin/ignite-web-agent.sh +++ b/modules/web-console/web-agent/bin/ignite-web-agent.sh @@ -16,34 +16,30 @@ # limitations under the License. # -SOURCE=$(dirname "$0") - -source "${SOURCE}"/include/functions.sh - -# -# Discover path to Java executable and check it's version. -# -checkJava - -# -# Set IGNITE_HOME. -# -export IGNITE_HOME="$(dirname "$(cd "$(dirname "$0")"; "pwd")")"; - -DIR="$( dirname "$SOURCE" )" +SOURCE="${BASH_SOURCE[0]}" +# Resolve $SOURCE until the file is no longer a symlink. while [ -h "$SOURCE" ] do - SOURCE="$(readlink "$SOURCE")" + IGNITE_HOME="$(cd -P "$( dirname "$SOURCE" )" && pwd)" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + SOURCE="$(readlink "$SOURCE")" - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + # If $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located. + [[ $SOURCE != /* ]] && SOURCE="$IGNITE_HOME/$SOURCE" done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# +# Set IGNITE_HOME. +# +export IGNITE_HOME="$(cd -P "$( dirname "$SOURCE" )" && pwd)" + +source "${IGNITE_HOME}"/include/functions.sh -cd $DIR +# +# Discover path to Java executable and check it's version. +# +checkJava # # JVM options. See http://java.sun.com/javase/technologies/hotspot/vmoptions.jsp for more details. @@ -94,4 +90,4 @@ elif [ $version -eq 11 ] ; then ${JVM_OPTS}" fi -"$JAVA" ${JVM_OPTS} -cp "*" org.apache.ignite.console.agent.AgentLauncher "$@" +"$JAVA" ${JVM_OPTS} -cp "${IGNITE_HOME}/*" org.apache.ignite.console.agent.AgentLauncher "$@" From 40a83a8e01f4af6981e1532ca86ab87458b38ffa Mon Sep 17 00:00:00 2001 From: Dmitrii Ryabov Date: Thu, 1 Nov 2018 11:51:58 +0300 Subject: [PATCH 063/403] IGNITE-9986 TcpDiscoverySelfTest.testNodeShutdownOnRingMessageWorkerStartNotFinished is flaky - Fixes #5082. Signed-off-by: Alexey Goncharuk --- .../main/java/org/apache/ignite/internal/IgnitionEx.java | 8 +++++--- .../org/apache/ignite/spi/discovery/tcp/ServerImpl.java | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index 1e56cde9e052a..d3dde71d22d5e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -1157,9 +1157,11 @@ private static T2 start0(GridStartContext startCtx try { grid.start(startCtx); } - catch (IgniteInterruptedCheckedException e) { - if (grid.starterThreadInterrupted) - Thread.interrupted(); + catch (Exception e) { + if (X.hasCause(e, IgniteInterruptedCheckedException.class, InterruptedException.class)) { + if (grid.starterThreadInterrupted) + Thread.interrupted(); + } throw e; } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 7b0ad53e1e972..15917f4c5e454 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -952,10 +952,10 @@ private void joinTopology() throws IgniteSpiException { timeout = threshold - U.currentTimeMillis(); } - catch (InterruptedException ignored) { + catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new IgniteSpiException("Thread has been interrupted."); + throw new IgniteSpiException("Thread has been interrupted.", e); } } From 9019e18ededf7641ed40ff0d0110bc4c3f575259 Mon Sep 17 00:00:00 2001 From: Maxim Muzafarov Date: Fri, 26 Oct 2018 15:16:40 +0300 Subject: [PATCH 064/403] IGNITE-7196 Restore binary state before node join to topology - Fixes #4520. Signed-off-by: Dmitriy Govorukhin --- .../apache/ignite/internal/IgniteKernal.java | 2 + .../pagemem/store/IgnitePageStoreManager.java | 12 +- .../wal/IgniteWriteAheadLogManager.java | 7 +- .../wal/record/MemoryRecoveryRecord.java | 7 +- .../processors/cache/GridCacheProcessor.java | 2 +- .../GridDhtPartitionsExchangeFuture.java | 76 ++---- .../cache/mvcc/MvccProcessorImpl.java | 18 +- .../DatabaseLifecycleListener.java | 13 +- .../GridCacheDatabaseSharedManager.java | 256 +++++++++++++----- .../IgniteCacheDatabaseSharedManager.java | 87 ++++-- .../file/FilePageStoreManager.java | 37 ++- .../persistence/metastorage/MetaStorage.java | 5 - .../wal/FileWriteAheadLogManager.java | 27 +- .../FsyncModeFileWriteAheadLogManager.java | 27 +- .../IgnitePdsDiskErrorsRecoveringTest.java | 12 +- .../persistence/db/wal/WalCompactionTest.java | 15 + .../pagemem/NoOpPageStoreManager.java | 6 + .../persistence/pagemem/NoOpWALManager.java | 5 - .../db/wal/IgniteWalRecoveryTest.java | 123 +++++++++ 19 files changed, 502 insertions(+), 235 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 1f9a5e8f1bfd3..250fbd78d3a54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1046,6 +1046,8 @@ public void start( fillNodeAttributes(clusterProc.updateNotifierEnabled()); ctx.cache().context().database().notifyMetaStorageSubscribersOnReadyForRead(); + + ctx.cache().context().database().startMemoryRestore(ctx); } catch (Throwable e) { U.error( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java index d7c61e9b114ff..1408383be168e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import java.util.Map; +import java.util.function.Predicate; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.pagemem.PageMemory; @@ -243,10 +244,19 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException; /** - * Cleanup persistent space for all caches. + * Cleanup persistent space for all caches except metastore. */ public void cleanupPersistentSpace() throws IgniteCheckedException; + /** + * Cleanup cache store whether it matches the provided predicate and if matched + * store was previously initizlized. + * + * @param cacheGrpPred Predicate to match by id cache group stores to clean. + * @param cleanFiles {@code True} to delete all persisted files related to particular store. + */ + public void cleanupPageStoreIfMatch(Predicate cacheGrpPred, boolean cleanFiles); + /** * Creates and initializes cache work directory retrieved from {@code cacheCfg}. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java index 68428d08a4c91..679eec98acc01 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java @@ -47,6 +47,8 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni /** * Resumes logging after start. When WAL manager is started, it will skip logging any updates until this * method is called to avoid logging changes induced by the state restore procedure. + * + * @throws IgniteCheckedException If fails. */ public void resumeLogging(WALPointer lastWrittenPtr) throws IgniteCheckedException; @@ -178,9 +180,4 @@ public WALPointer log(WALRecord entry, RolloverType rolloverType) * @param grpId Group id. */ public boolean disabled(int grpId); - - /** - * Cleanup all directories relating to WAL (e.g. work WAL dir, archive WAL dir). - */ - public void cleanupWalDirectories() throws IgniteCheckedException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java index 8843eeedef90e..92658cc4ce788 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java @@ -20,8 +20,13 @@ import org.apache.ignite.internal.util.typedef.internal.S; /** - * Marker that we start memory recovering + * Marker that we start memory recovering. + * + * @deprecated Previously, used to track node started\stopped states. But in fact only + * mark files created by method GridCacheDatabaseSharedManager#nodeStart(WALPointer) + * used. Should be removed in 3.0 release. */ +@Deprecated public class MemoryRecoveryRecord extends WALRecord { /** Create timestamp, millis */ private long time; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 1a169130b7736..ca0da8807c2c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2263,7 +2263,7 @@ private boolean checkForAffinityNode( * @param affNode {@code true} if it is affinity node for cache. * @throws IgniteCheckedException if failed. */ - private void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException { + public void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException { if (sharedCtx.pageStore() != null && affNode) initializationProtector.protect( desc.groupDescriptor().groupId(), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index ef704aa6576db..9933e11994d07 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -50,7 +50,6 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException; import org.apache.ignite.internal.IgniteDiagnosticAware; @@ -83,7 +82,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.GridCacheUtils; -import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext; import org.apache.ignite.internal.processors.cache.StateChangeRequest; import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter; @@ -872,13 +870,22 @@ else if (msg instanceof WalStateAbstractMessage) * @throws IgniteCheckedException If failed. */ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedException { - boolean baselineNode = isLocalNodeInBaseline(); - - if (!baselineNode) { + if (!isLocalNodeInBaseline()) { cctx.exchange().exchangerBlockingSectionBegin(); try { - cctx.cache().cleanupCachesDirectories(); + cctx.database().cleanupRestoredCaches(); + + for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) { + if (CU.isPersistentCache(desc.cacheConfiguration(), + cctx.gridConfig().getDataStorageConfiguration())) { + // Perform cache init from scratch. + cctx.cache().preparePageStore(desc, true); + } + } + + // Set initial node started marker. + cctx.database().nodeStart(null); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -894,34 +901,11 @@ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedExce cctx.exchange().exchangerBlockingSectionEnd(); } - LocalJoinCachesContext locJoinCtx = exchActions == null ? null : exchActions.localJoinContext(); - - List> caches = locJoinCtx == null ? null : - locJoinCtx.caches(); - - if (!cctx.kernalContext().clientNode()) { - List startDescs = new ArrayList<>(); - - if (caches != null) { - for (T2 c : caches) { - DynamicCacheDescriptor startDesc = c.get1(); - - if (CU.isPersistentCache(startDesc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration())) - startDescs.add(startDesc); - } - } - - cctx.exchange().exchangerBlockingSectionBegin(); - - try { - cctx.database().readCheckpointAndRestoreMemory(startDescs, !baselineNode); - } - finally { - cctx.exchange().exchangerBlockingSectionEnd(); - } - } + if (!cctx.kernalContext().clientNode()) + cctx.database().onDoneRestoreBinaryMemory(); - IgniteInternalFuture cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), locJoinCtx); + IgniteInternalFuture cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), + exchActions == null ? null : exchActions.localJoinContext()); ensureClientCachesStarted(); @@ -1082,26 +1066,8 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) { cctx.exchange().exchangerBlockingSectionEnd(); } - if (!cctx.kernalContext().clientNode()) { - List startDescs = new ArrayList<>(); - - for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) { - DynamicCacheDescriptor desc = startReq.descriptor(); - - if (CU.isPersistentCache(desc.cacheConfiguration(), - cctx.gridConfig().getDataStorageConfiguration())) - startDescs.add(desc); - } - - cctx.exchange().exchangerBlockingSectionBegin(); - - try { - cctx.database().readCheckpointAndRestoreMemory(startDescs, false); - } - finally { - cctx.exchange().exchangerBlockingSectionEnd(); - } - } + if (!cctx.kernalContext().clientNode()) + cctx.database().onDoneRestoreBinaryMemory(); assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started."; @@ -2150,9 +2116,9 @@ public void finishMerged() { cctx.database().releaseHistoryForExchange(); - cctx.database().rebuildIndexesIfNeeded(this); - if (err == null) { + cctx.database().rebuildIndexesIfNeeded(this); + for (CacheGroupContext grp : cctx.cache().cacheGroups()) { if (!grp.isLocal()) grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java index 4fada489306a5..7a4fa39aaea6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java @@ -324,10 +324,22 @@ public MvccProcessorImpl(GridKernalContext ctx) { } /** {@inheritDoc} */ - @SuppressWarnings("ConstantConditions") - @Override public void beforeMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + txLogPageStoreInit(mgr); + } + + /** {@inheritDoc} */ + @Override public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + // In case of blt changed we should re-init TX_LOG cache. + txLogPageStoreInit(mgr); + } + + /** + * @param mgr Database shared manager. + * @throws IgniteCheckedException If failed. + */ + private void txLogPageStoreInit(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { assert CU.isPersistenceEnabled(ctx.config()); - assert txLog == null; ctx.cache().context().pageStore().initialize(TX_LOG_CACHE_ID, 1, TX_LOG_CACHE_NAME, mgr.dataRegion(TX_LOG_CACHE_NAME).memoryMetrics()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java index f96cdd91a7e8f..ae65c77cc6e96 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java @@ -23,7 +23,6 @@ * */ public interface DatabaseLifecycleListener { - /** * @param mgr Database shared manager. * @@ -31,10 +30,16 @@ public interface DatabaseLifecycleListener { void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; /** - * @param mgr Page store manager. - * + * @param mgr Database shared manager. + * @throws IgniteCheckedException If failed. + */ + public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + + /** + * @param mgr Database shared manager. + * @throws IgniteCheckedException If failed. */ - void beforeMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; /** * @param mgr Database shared manager. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index ed44f56ec1582..089ccb6d4c077 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -53,6 +53,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -96,7 +97,6 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; -import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; @@ -159,8 +159,8 @@ import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.mxbean.DataStorageMetricsMXBean; import org.apache.ignite.thread.IgniteThread; -import org.jetbrains.annotations.NotNull; import org.apache.ignite.thread.IgniteThreadPoolExecutor; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentLinkedHashMap; @@ -168,6 +168,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME; import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD; @@ -282,6 +283,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** */ private boolean stopping; + /** + * The position of last seen WAL pointer. Used for resumming logging from this pointer. + * + * If binary memory recovery pefrormed on node start, the checkpoint END pointer will store + * not the last WAL pointer and can't be used for resumming logging. + */ + private volatile WALPointer walTail; + /** Checkpoint runner thread pool. If null tasks are to be run in single thread */ @Nullable private IgniteThreadPoolExecutor asyncRunner; @@ -330,7 +339,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** Number of pages in current checkpoint at the beginning of checkpoint. */ private volatile int currCheckpointPagesCnt; - /** */ + /** + * MetaStorage instance. Value {@code null} means storage not initialized yet. + * Guarded by {@link GridCacheDatabaseSharedManager#checkpointReadLock()} + */ private MetaStorage metaStorage; /** */ @@ -434,6 +446,8 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { } /** + * Create metastorage data region configuration with enabled persistence by default. + * * @param storageCfg Data storage configuration. * @return Data region configuration. */ @@ -520,10 +534,43 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu } } - /** - * Cleanup checkpoint directory. - */ + /** {@inheritDoc} */ + @Override public void cleanupRestoredCaches() { + if (dataRegionMap == null) + return; + + for (CacheGroupDescriptor grpDesc : cctx.cache().cacheGroupDescriptors().values()) { + String regionName = grpDesc.config().getDataRegionName(); + + DataRegion region = dataRegionMap.get(regionName == null ? DFLT_DATA_REG_DEFAULT_NAME : regionName); + + if (region == null) + continue; + + int partitions = grpDesc.config().getAffinity().partitions(); + + if (region.pageMemory() instanceof PageMemoryEx) { + PageMemoryEx memEx = (PageMemoryEx)region.pageMemory(); + + for (int partId = 0; partId < partitions; partId++) + memEx.invalidate(grpDesc.groupId(), partId); + } + } + + storeMgr.cleanupPageStoreIfMatch( + new Predicate() { + @Override public boolean test(Integer grpId) { + return MetaStorage.METASTORAGE_CACHE_ID != grpId; + } + }, + true); + } + + /** {@inheritDoc} */ @Override public void cleanupCheckpointDirectory() throws IgniteCheckedException { + if (cpHistory != null) + cpHistory = new CheckpointHistory(cctx.kernalContext()); + try { try (DirectoryStream files = Files.newDirectoryStream(cpDir.toPath())) { for (Path path : files) @@ -645,7 +692,7 @@ private void readMetastore() throws IgniteCheckedException { checkpointReadLock(); try { - restoreMemory(status, true, storePageMem, false); + restoreMemory(status, true, storePageMem, Collections.emptySet()); metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true); @@ -658,12 +705,18 @@ private void readMetastore() throws IgniteCheckedException { notifyMetastorageReadyForRead(); } finally { - checkpointReadUnlock(); - } + metaStorage = null; + + storePageMem.stop(true); - metaStorage = null; + cctx.pageStore().cleanupPageStoreIfMatch(new Predicate() { + @Override public boolean test(Integer grpId) { + return MetaStorage.METASTORAGE_CACHE_ID == grpId; + } + }, false); - storePageMem.stop(true); + checkpointReadUnlock(); + } } catch (StorageException e) { cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); @@ -680,7 +733,7 @@ private void readMetastore() throws IgniteCheckedException { snapshotMgr = cctx.snapshot(); - if (!cctx.localNode().isClient()) { + if (!cctx.kernalContext().clientNode()) { initDataBase(); registrateMetricsMBean(); @@ -789,11 +842,8 @@ private void unRegistrateMetricsMBean() { } /** {@inheritDoc} */ - @Override public void readCheckpointAndRestoreMemory( - List cachesToStart, - boolean restoreMetastorageOnly - ) throws IgniteCheckedException { - assert !cctx.localNode().isClient(); + @Override public void onDoneRestoreBinaryMemory() throws IgniteCheckedException { + assert !cctx.kernalContext().clientNode(); long time = System.currentTimeMillis(); @@ -801,50 +851,88 @@ private void unRegistrateMetricsMBean() { try { for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) - lsnr.beforeMemoryRestore(this); + lsnr.beforeResumeWalLogging(this); - if (!F.isEmpty(cachesToStart)) { - for (DynamicCacheDescriptor desc : cachesToStart) { - if (CU.affinityNode(cctx.localNode(), desc.cacheConfiguration().getNodeFilter())) - storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration())); - } - } + cctx.pageStore().initializeForMetastorage(); CheckpointStatus status = readCheckpointStatus(); - cctx.pageStore().initializeForMetastorage(); + // Binary memory should be recovered at startup. + assert !status.needRestoreMemory() : status; + + WALPointer statusEndPtr = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr; + + // If binary memory recovery occurs resume from the last walTail in the other case from END checkpoint. + WALPointer walPtr = walTail == null ? statusEndPtr : walTail; + + cctx.wal().resumeLogging(walPtr); + + walTail = null; metaStorage = new MetaStorage( cctx, dataRegionMap.get(METASTORE_DATA_REGION_NAME), - (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME) + (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME), + false ); - WALPointer restore = restoreMemory(status, restoreMetastorageOnly, (PageMemoryEx) metaStorage.pageMemory(), true); + // Init metastore only after WAL logging resumed. Can't do it earlier because + // MetaStorage first initialization also touches WAL, look at #isWalDeltaRecordNeeded. + metaStorage.init(this); - if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { - throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " + - status.endPtr + ". Can't restore memory - critical part of WAL archive is missing."); - } + notifyMetastorageReadyForReadWrite(); - // First, bring memory to the last consistent checkpoint state if needed. - // This method should return a pointer to the last valid record in the WAL. - cctx.wal().resumeLogging(restore); + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) + lsnr.afterMemoryRestore(this); + } + catch (IgniteCheckedException e) { + if (X.hasCause(e, StorageException.class, IOException.class)) + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); - WALPointer ptr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis())); + throw e; + } + finally { + checkpointReadUnlock(); - if (ptr != null) { - cctx.wal().flush(ptr, true); + U.log(log, "Resume logging performed in " + (System.currentTimeMillis() - time) + " ms."); + } + } - nodeStart(ptr); - } + /** + * @param cacheGrps Cache groups to restore. + * @return Last seen WAL pointer during binary memory recovery. + * @throws IgniteCheckedException If failed. + */ + protected WALPointer restoreBinaryMemory(Set cacheGrps) throws IgniteCheckedException { + assert !cctx.kernalContext().clientNode(); - metaStorage.init(this); + long time = System.currentTimeMillis(); - notifyMetastorageReadyForReadWrite(); + checkpointReadLock(); + try { for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) - lsnr.afterMemoryRestore(this); + lsnr.beforeBinaryMemoryRestore(this); + + cctx.pageStore().initializeForMetastorage(); + + CheckpointStatus status = readCheckpointStatus(); + + // First, bring memory to the last consistent checkpoint state if needed. + // This method should return a pointer to the last valid record in the WAL. + WALPointer tailWalPtr = restoreMemory(status, + false, + (PageMemoryEx)dataRegionMap.get(METASTORE_DATA_REGION_NAME).pageMemory(), + cacheGrps); + + if (tailWalPtr == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { + throw new StorageException("Restore wal pointer = " + tailWalPtr + ", while status.endPtr = " + + status.endPtr + ". Can't restore memory - critical part of WAL archive is missing."); + } + + nodeStart(tailWalPtr); + + return tailWalPtr; } catch (IgniteCheckedException e) { if (X.hasCause(e, StorageException.class, IOException.class)) @@ -860,14 +948,9 @@ private void unRegistrateMetricsMBean() { } } - /** - * Creates file with current timestamp and specific "node-started.bin" suffix - * and writes into memory recovery pointer. - * - * @param ptr Memory recovery wal pointer. - */ - private void nodeStart(WALPointer ptr) throws IgniteCheckedException { - FileWALPointer p = (FileWALPointer)ptr; + /** {@inheritDoc} */ + @Override public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException { + FileWALPointer p = (FileWALPointer)(ptr == null ? CheckpointStatus.NULL_PTR : ptr); String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX; String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX; @@ -1947,11 +2030,33 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC } } + /** {@inheritDoc} */ + @Override public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedException { + if (kctx.clientNode()) + return; + + // Preform early regions startup before restoring state. + initAndStartRegions(kctx.config().getDataStorageConfiguration()); + + // Only presistence caches to start. + for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) { + if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration())) + storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration())); + } + + final WALPointer restoredPtr = restoreBinaryMemory(cctx.cache().cacheGroupDescriptors().keySet()); + + walTail = restoredPtr; + + if (restoredPtr != null) + U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restoredPtr + ']'); + } + /** * @param status Checkpoint status. * @param metastoreOnly If {@code True} restores Metastorage only. * @param storePageMem Metastore page memory. - * @param finalizeCp If {@code True}, finalizes checkpoint on recovery. + * @param cacheGrps Cache groups to restore. * @throws IgniteCheckedException If failed. * @throws StorageException In case I/O error occurred during operations with storage. */ @@ -1959,7 +2064,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC CheckpointStatus status, boolean metastoreOnly, PageMemoryEx storePageMem, - boolean finalizeCp + Set cacheGrps ) throws IgniteCheckedException { assert !metastoreOnly || storePageMem != null; @@ -1984,8 +2089,14 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, log); - Collection ignoreGrps = metastoreOnly ? Collections.emptySet() : - F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps); + // Always perform recovery at least meta storage cache. + Set restoreGrps = new HashSet<>(Collections.singletonList(METASTORAGE_CACHE_ID)); + + if (!metastoreOnly && !F.isEmpty(cacheGrps)) { + restoreGrps.addAll(cacheGrps.stream() + .filter(g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g)) + .collect(Collectors.toSet())); + } int applied = 0; @@ -2005,10 +2116,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC // several repetitive restarts and the same pages may have changed several times. int grpId = pageRec.fullPageId().groupId(); - if (metastoreOnly && grpId != METASTORAGE_CACHE_ID) - continue; - - if (!ignoreGrps.contains(grpId)) { + if (restoreGrps.contains(grpId)) { long pageId = pageRec.fullPageId().pageId(); PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId); @@ -2041,10 +2149,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC { int grpId = metaStateRecord.groupId(); - if (metastoreOnly && grpId != METASTORAGE_CACHE_ID) - continue; - - if (ignoreGrps.contains(grpId)) + if (!restoreGrps.contains(grpId)) continue; int partId = metaStateRecord.partitionId(); @@ -2065,10 +2170,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC { int grpId = destroyRecord.groupId(); - if (metastoreOnly && grpId != METASTORAGE_CACHE_ID) - continue; - - if (ignoreGrps.contains(grpId)) + if (!restoreGrps.contains(grpId)) continue; PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId); @@ -2086,10 +2188,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC int grpId = r.groupId(); - if (metastoreOnly && grpId != METASTORAGE_CACHE_ID) - continue; - - if (!ignoreGrps.contains(grpId)) { + if (restoreGrps.contains(grpId)) { long pageId = r.pageId(); PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId); @@ -2119,7 +2218,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC } } - if (!finalizeCp) + if (metastoreOnly) return null; WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); @@ -2249,6 +2348,7 @@ else if (log != null) /** * @param status Last registered checkpoint status. + * @param metastoreOnly If {@code True} only records related to metastorage will be processed. * @throws IgniteCheckedException If failed to apply updates. * @throws StorageException If IO exception occurred while reading write-ahead log. */ @@ -2289,9 +2389,13 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th for (DataEntry dataEntry : dataRec.writeEntries()) { int cacheId = dataEntry.cacheId(); - int grpId = cctx.cache().cacheDescriptor(cacheId).groupId(); + DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptor(cacheId); - if (!ignoreGrps.contains(grpId)) { + // Can empty in case recovery node on blt changed. + if (cacheDesc == null) + continue; + + if (!ignoreGrps.contains(cacheDesc.groupId())) { GridCacheContext cacheCtx = cctx.cacheContext(cacheId); applyUpdate(cacheCtx, dataEntry); @@ -2789,8 +2893,9 @@ private static String checkpointFileName(long cpTs, UUID cpId, CheckpointEntryTy /** * @param cp Checkpoint entry. * @param type Checkpoint type. + * @return Checkpoint file name. */ - private static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) { + public static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) { return checkpointFileName(cp.timestamp(), cp.checkpointId(), type); } @@ -4204,7 +4309,8 @@ private CheckpointStatus(long cpStartTs, UUID cpStartId, WALPointer startPtr, UU } /** - * @return {@code True} if need to apply page log to restore tree structure. + * @return {@code True} if need perform binary memory recovery. Only records {@link PageDeltaRecord} + * and {@link PageSnapshot} needs to be applyed from {@link #cpStartId}. */ public boolean needRestoreMemory() { return !F.eq(cpStartId, cpEndId) && !F.eq(NULL_UUID, cpStartId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index db6b987bc7b68..589b495bbf317 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -37,6 +37,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider; @@ -45,7 +46,6 @@ import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.processors.cache.CacheGroupContext; -import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; @@ -99,6 +99,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** */ private volatile boolean dataRegionsInitialized; + /** */ + private volatile boolean dataRegionsStarted; + /** */ protected Map memMetricsMap; @@ -238,6 +241,8 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe initDataRegions0(memCfg); dataRegionsInitialized = true; + + U.log(log, "Configured data regions initialized successfully [total=" + dataRegionMap.size() + ']'); } /** @@ -648,14 +653,19 @@ public DataStorageMetrics persistentStoreMetrics() { } /** - * @param cachesToStart Started caches. - * @param restoreMetastorageOnly Apply updates only for metastorage. - * @throws IgniteCheckedException If failed. + * @throws IgniteCheckedException If fails. */ - public void readCheckpointAndRestoreMemory( - List cachesToStart, - boolean restoreMetastorageOnly - ) throws IgniteCheckedException { + public void onDoneRestoreBinaryMemory() throws IgniteCheckedException { + // No-op. + } + + /** + * Creates file with current timestamp and specific "node-started.bin" suffix + * and writes into memory recovery pointer. + * + * @param ptr Memory recovery wal pointer. + */ + public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException { // No-op. } @@ -781,7 +791,15 @@ public void checkpointReadLockTimeout(long val) { } /** - * No-op for non-persistent storage. + * Method will perform cleanup cache page memory and each cache partition store. + */ + public void cleanupRestoredCaches() { + // No-op. + } + + /** + * Clean checkpoint directory {@link GridCacheDatabaseSharedManager#cpDir}. The operation + * is necessary when local node joined to baseline topology with different consistentId. */ public void cleanupCheckpointDirectory() throws IgniteCheckedException { // No-op. @@ -835,6 +853,16 @@ public boolean beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws I return false; } + /** + * Perform memory restore before {@link GridDiscoveryManager} start. + * + * @param kctx Current kernal context. + * @throws IgniteCheckedException If fails. + */ + public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedException { + // No-op. + } + /** * Called when all partitions have been fully restored and pre-created on node start. * @@ -1138,23 +1166,46 @@ protected File buildPath(String path, String consId) throws IgniteCheckedExcepti /** {@inheritDoc} */ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { - if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null) + if (kctx.clientNode() && kctx.config().getDataStorageConfiguration() == null) return; - DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration(); + initAndStartRegions(kctx.config().getDataStorageConfiguration()); - assert memCfg != null; + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) + lsnr.afterInitialise(this); + } - initDataRegions(memCfg); + /** + * @param cfg Current data storage configuration. + * @throws IgniteCheckedException If fails. + */ + protected void initAndStartRegions(DataStorageConfiguration cfg) throws IgniteCheckedException { + assert cfg != null; + + initDataRegions(cfg); + + startDataRegions(cfg); + } + + /** + * @param cfg Regions configuration. + * @throws IgniteCheckedException If fails. + */ + private void startDataRegions(DataStorageConfiguration cfg) throws IgniteCheckedException { + if (dataRegionsStarted) + return; + + assert cfg != null; registerMetricsMBeans(); startMemoryPolicies(); - initPageMemoryDataStructures(memCfg); + initPageMemoryDataStructures(cfg); - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) - lsnr.afterInitialise(this); + dataRegionsStarted = true; + + U.log(log, "Configured data regions started successfully [total=" + dataRegionMap.size() + ']'); } /** {@inheritDoc} */ @@ -1163,7 +1214,7 @@ protected File buildPath(String path, String consId) throws IgniteCheckedExcepti } /** - * @param shutdown Shutdown. + * @param shutdown {@code True} to force memory regions shutdown. */ private void onDeActivate(boolean shutdown) { for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) @@ -1189,6 +1240,8 @@ private void onDeActivate(boolean shutdown) { } dataRegionsInitialized = false; + + dataRegionsStarted = false; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index b6cb163e738bd..0f01fd452268f 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -32,11 +32,14 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.configuration.CacheConfiguration; @@ -228,15 +231,29 @@ public FilePageStoreManager(GridKernalContext ctx) { } } + /** {@inheritDoc} */ + @Override public void cleanupPageStoreIfMatch(Predicate cacheGrpPred, boolean cleanFiles) { + Map filteredStores = idxCacheStores.entrySet().stream() + .filter(e -> cacheGrpPred.test(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + IgniteCheckedException ex = shutdown(filteredStores.values(), cleanFiles); + + if (ex != null) + U.error(log, "Failed to gracefully stop page store managers", ex); + + idxCacheStores.entrySet().removeIf(e -> cacheGrpPred.test(e.getKey())); + + U.log(log, "Cleanup cache stores [total=" + filteredStores.keySet().size() + + ", left=" + idxCacheStores.size() + ", cleanFiles=" + cleanFiles + ']'); + } + /** {@inheritDoc} */ @Override public void stop0(boolean cancel) { if (log.isDebugEnabled()) log.debug("Stopping page store manager."); - IgniteCheckedException ex = shutdown(false); - - if (ex != null) - U.error(log, "Failed to gracefully stop page store manager", ex); + cleanupPageStoreIfMatch(p -> true, false); } /** {@inheritDoc} */ @@ -255,8 +272,6 @@ public FilePageStoreManager(GridKernalContext ctx) { " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); stop0(true); - - idxCacheStores.clear(); } /** {@inheritDoc} */ @@ -289,6 +304,8 @@ public FilePageStoreManager(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void initialize(int cacheId, int partitions, String workingDir, AllocatedPageTracker tracker) throws IgniteCheckedException { + assert storeWorkDir != null; + if (!idxCacheStores.containsKey(cacheId)) { CacheStoreHolder holder = initDir( new File(storeWorkDir, workingDir), @@ -306,6 +323,8 @@ public FilePageStoreManager(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData) throws IgniteCheckedException { + assert storeWorkDir != null; + int grpId = grpDesc.groupId(); if (!idxCacheStores.containsKey(grpId)) { @@ -319,6 +338,8 @@ public FilePageStoreManager(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void initializeForMetastorage() throws IgniteCheckedException { + assert storeWorkDir != null; + int grpId = MetaStorage.METASTORAGE_CACHE_ID; if (!idxCacheStores.containsKey(grpId)) { @@ -900,10 +921,10 @@ public File cacheWorkDir(boolean isSharedGroup, String cacheOrGroupName) { /** * @param cleanFiles {@code True} if the stores should delete it's files upon close. */ - private IgniteCheckedException shutdown(boolean cleanFiles) { + private IgniteCheckedException shutdown(Collection holders, boolean cleanFiles) { IgniteCheckedException ex = null; - for (CacheStoreHolder holder : idxCacheStores.values()) + for (CacheStoreHolder holder : holders) ex = shutdown(holder, cleanFiles, ex); return ex; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java index 3981d4d906d11..4a243aaea005a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java @@ -135,11 +135,6 @@ public MetaStorage( this.failureProcessor = cctx.kernalContext().failure(); } - /** */ - public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) { - this(cctx, memPlc, memMetrics, false); - } - /** */ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException { getOrAllocateMetas(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index b637a1dd53394..31dbb968e9933 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -33,10 +33,8 @@ import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.ClosedByInterruptException; -import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; -import java.nio.file.Path; import java.sql.Time; import java.util.ArrayList; import java.util.Arrays; @@ -689,6 +687,8 @@ private void checkWalConfiguration() throws IgniteCheckedException { @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException { assert currHnd == null; assert lastPtr == null || lastPtr instanceof FileWALPointer; + assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) : + "Trying to restore FileWriteHandle on deactivated write ahead log manager"; FileWALPointer filePtr = (FileWALPointer)lastPtr; @@ -1486,29 +1486,6 @@ private void checkOrPrepareFiles() throws StorageException { checkFiles(0, false, null, null); } - /** {@inheritDoc} */ - @Override public void cleanupWalDirectories() throws IgniteCheckedException { - try { - try (DirectoryStream files = Files.newDirectoryStream(walWorkDir.toPath())) { - for (Path path : files) - Files.delete(path); - } - } - catch (IOException e) { - throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e); - } - - try { - try (DirectoryStream files = Files.newDirectoryStream(walArchiveDir.toPath())) { - for (Path path : files) - Files.delete(path); - } - } - catch (IOException e) { - throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e); - } - } - /** * Clears whole the file, fills with zeros for Default mode. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index d1e0ebc3e1e33..def9bf20a7c3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -28,10 +28,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; -import java.nio.file.Path; import java.sql.Time; import java.util.ArrayList; import java.util.Arrays; @@ -539,6 +537,8 @@ private void checkWalConfiguration() throws IgniteCheckedException { @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException { assert currentHnd == null; assert lastPtr == null || lastPtr instanceof FileWALPointer; + assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) : + "Trying to restore FileWriteHandle on deactivated write ahead log manager"; FileWALPointer filePtr = (FileWALPointer)lastPtr; @@ -1022,29 +1022,6 @@ private FileDescriptor[] walArchiveFiles() { return cctx.walState().isDisabled(grpId); } - /** {@inheritDoc} */ - @Override public void cleanupWalDirectories() throws IgniteCheckedException { - try { - try (DirectoryStream files = Files.newDirectoryStream(walWorkDir.toPath())) { - for (Path path : files) - Files.delete(path); - } - } - catch (IOException e) { - throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e); - } - - try { - try (DirectoryStream files = Files.newDirectoryStream(walArchiveDir.toPath())) { - for (Path path : files) - Files.delete(path); - } - } - catch (IOException e) { - throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e); - } - } - /** * Lists files in archive directory and returns the index of last archived file. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java index bd30696a1e260..a214fc3fbd1b5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java @@ -25,7 +25,6 @@ import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; @@ -177,15 +176,18 @@ public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception { boolean activationFailed = false; try { grid = startGrid(0); - grid.cluster().active(true); } - catch (IgniteException e) { - log.warning("Activation test exception", e); + catch (IgniteCheckedException e) { + boolean interrupted = Thread.interrupted(); + + if (interrupted) + log.warning("Ignore interrupted excpetion [" + + "thread=" + Thread.currentThread().getName() + ']', e); activationFailed = true; } - Assert.assertTrue("Activation must be failed", activationFailed); + Assert.assertTrue("Ignite instance startup must be failed", activationFailed); // Grid should be automatically stopped after checkpoint fail. awaitStop(grid); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java index dcc2280f8cee0..3374860baa7c1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java @@ -21,6 +21,7 @@ import java.io.RandomAccessFile; import java.util.Arrays; import java.util.Comparator; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; @@ -221,6 +222,20 @@ else if (arr[i] != 1) { } assertFalse(fail); + + // Check compation successfully reset on blt changed. + stopAllGrids(); + + Ignite ignite = startGrids(2); + + ignite.cluster().active(true); + + resetBaselineTopology(); + + // This node will join to different blt. + startGrid(2); + + awaitPartitionMapExchange(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java index 39c7dc92d2e43..44071eaa93e07 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java @@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.GridKernalContext; @@ -234,6 +235,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { // No-op. } + /** {@inheritDoc} */ + @Override public void cleanupPageStoreIfMatch(Predicate cacheGrpPred, boolean cleanFiles) { + // No-op. + } + /** {@inheritDoc} */ @Override public boolean checkAndInitCacheWorkDir(CacheConfiguration cacheCfg) throws IgniteCheckedException { return false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java index 0beeed783a007..ea3ed2f8ae0f9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java @@ -107,11 +107,6 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { return false; } - /** {@inheritDoc} */ - @Override public void cleanupWalDirectories() throws IgniteCheckedException { - // No-op. - } - /** {@inheritDoc} */ @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException { // No-op. diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java index c2b5dc2d382ef..b56b695671269 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java @@ -18,8 +18,12 @@ package org.apache.ignite.internal.processors.cache.persistence.db.wal; import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -38,6 +42,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteCompute; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; @@ -53,9 +58,11 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.DiscoverySpiTestListener; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi; import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.pagemem.wal.WALIterator; @@ -67,15 +74,22 @@ import org.apache.ignite.internal.pagemem.wal.record.TxRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord; +import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.CA; +import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.internal.util.typedef.PAX; @@ -95,6 +109,8 @@ import org.apache.ignite.transactions.TransactionIsolation; import org.junit.Assert; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; /** @@ -116,6 +132,9 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest { /** */ private static final String RENAMED_CACHE_NAME = "partitioned0"; + /** */ + private static final String CACHE_TO_DESTROY_NAME = "destroyCache"; + /** */ private static final String LOC_CACHE_NAME = "local"; @@ -466,6 +485,110 @@ public void testWalLargeValue() throws Exception { } } + /** + * Check binary recover completes successfully when node stopped at the middle of checkpoint. + * Destroy cache_data.bin file for particular cache to emulate missing {@link DynamicCacheDescriptor} + * file (binary recovery should complete successfully in this case). + * + * @throws Exception if failed. + */ + public void testBinaryRecoverBeforePMEWhenMiddleCheckpoint() throws Exception { + startGrids(3); + + IgniteEx ig2 = grid(2); + + ig2.cluster().active(true); + + IgniteCache cache = ig2.cache(CACHE_NAME); + + for (int i = 1; i <= 4_000; i++) + cache.put(i, new BigObject(i)); + + BigObject objToCheck; + + ig2.getOrCreateCache(CACHE_TO_DESTROY_NAME).put(1, objToCheck = new BigObject(1)); + + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig2 + .context().cache().context().database(); + + IgniteInternalFuture cpFinishFut = dbMgr.forceCheckpoint("force checkpoint").finishFuture(); + + // Delete checkpoint END file to emulate node stopped at the middle of checkpoint. + cpFinishFut.listen(new IgniteInClosureX() { + @Override public void applyx(IgniteInternalFuture fut0) throws IgniteCheckedException { + try { + CheckpointEntry cpEntry = dbMgr.checkpointHistory().lastCheckpoint(); + + String cpEndFileName = GridCacheDatabaseSharedManager.checkpointFileName(cpEntry, + CheckpointEntryType.END); + + Files.delete(Paths.get(dbMgr.checkpointDirectory().getAbsolutePath(), cpEndFileName)); + + log.info("Checkpoint marker removed [cpEndFileName=" + cpEndFileName + ']'); + } + catch (IOException e) { + throw new IgniteCheckedException(e); + } + } + }); + + // Resolve cache directory. Emulating cache destroy in the middle of checkpoint. + IgniteInternalCache destoryCache = ig2.cachex(CACHE_TO_DESTROY_NAME); + + FilePageStoreManager pageStoreMgr = (FilePageStoreManager)destoryCache.context().shared().pageStore(); + + File destroyCacheWorkDir = pageStoreMgr.cacheWorkDir(destoryCache.configuration()); + + // Stop the whole cluster + stopAllGrids(); + + // Delete cache_data.bin file for this cache. Binary recovery should complete successfully after it. + final File[] files = destroyCacheWorkDir.listFiles(new FilenameFilter() { + @Override public boolean accept(final File dir, final String name) { + return name.endsWith(CACHE_DATA_FILENAME); + } + }); + + assertTrue(files.length > 0); + + for (final File file : files) + assertTrue("Can't remove " + file.getAbsolutePath(), file.delete()); + + startGrids(2); + + // Preprare Ignite instance configuration with additional Discovery checks. + final String ig2Name = getTestIgniteInstanceName(2); + + final IgniteConfiguration onJoinCfg = optimize(getConfiguration(ig2Name)); + + // Check restore beeing called before PME and joining node to cluster. + ((IgniteDiscoverySpi)onJoinCfg.getDiscoverySpi()) + .setInternalListener(new DiscoverySpiTestListener() { + @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) { + String nodeName = locNode.attribute(ATTR_IGNITE_INSTANCE_NAME); + + GridCacheSharedContext sharedCtx = ((IgniteEx)ignite(getTestIgniteInstanceIndex(nodeName))) + .context() + .cache() + .context(); + + if (nodeName.equals(ig2Name)) { + // Checkpoint history initialized on node start. + assertFalse(((GridCacheDatabaseSharedManager)sharedCtx.database()) + .checkpointHistory().checkpoints().isEmpty()); + } + + super.beforeJoin(locNode, log); + } + }); + + Ignite restoredIg2 = startGrid(ig2Name, onJoinCfg); + + awaitPartitionMapExchange(); + + assertEquals(restoredIg2.cache(CACHE_TO_DESTROY_NAME).get(1), objToCheck); + } + /** * @throws Exception if failed. */ From cdfd2b314131d22211e4f44aaa2f0f5f67922d9e Mon Sep 17 00:00:00 2001 From: Maxim Muzafarov Date: Tue, 30 Oct 2018 00:12:36 +0300 Subject: [PATCH 065/403] IGNITE-10033 Update Grid(Ignite)CacheDatabaseSharedManager according to accepted coding guidelines - Fixes #5156. Signed-off-by: Dmitriy Govorukhin --- .../GridCacheDatabaseSharedManager.java | 135 ++++++---------- .../IgniteCacheDatabaseSharedManager.java | 150 ++++++++++-------- ...IgniteNoParrallelClusterIsAllowedTest.java | 2 +- 3 files changed, 134 insertions(+), 153 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 089ccb6d4c077..bf6464d3d4a91 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -57,7 +57,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.management.ObjectName; import org.apache.ignite.DataStorageMetrics; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; @@ -324,9 +323,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** */ private DataStorageMetricsImpl persStoreMetrics; - /** */ - private ObjectName persistenceMetricsMbeanName; - /** Counter for written checkpoint pages. Not null only if checkpoint is running. */ private volatile AtomicInteger writtenPagesCntr = null; @@ -438,7 +434,7 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { addDataRegion( memCfg, - createDataRegionConfiguration(memCfg), + createMetastoreDataRegionConfig(memCfg), false ); @@ -451,7 +447,7 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { * @param storageCfg Data storage configuration. * @return Data region configuration. */ - private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfiguration storageCfg) { + private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfiguration storageCfg) { DataRegionConfiguration cfg = new DataRegionConfiguration(); cfg.setName(METASTORE_DATA_REGION_NAME); @@ -667,7 +663,7 @@ private void readMetastore() throws IgniteCheckedException { try { DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration(); - DataRegionConfiguration plcCfg = createDataRegionConfiguration(memCfg); + DataRegionConfiguration plcCfg = createMetastoreDataRegionConfig(memCfg); File allocPath = buildAllocPath(plcCfg); @@ -733,10 +729,15 @@ private void readMetastore() throws IgniteCheckedException { snapshotMgr = cctx.snapshot(); - if (!cctx.kernalContext().clientNode()) { - initDataBase(); - - registrateMetricsMBean(); + if (!cctx.kernalContext().clientNode() && persistenceCfg.getCheckpointThreads() > 1) { + asyncRunner = new IgniteThreadPoolExecutor( + CHECKPOINT_RUNNER_THREAD_PREFIX, + cctx.igniteInstanceName(), + persistenceCfg.getCheckpointThreads(), + persistenceCfg.getCheckpointThreads(), + 30_000, + new LinkedBlockingQueue<>() + ); } if (checkpointer == null) @@ -759,61 +760,17 @@ private void readMetastore() throws IgniteCheckedException { stopping = false; } - /** - * - */ - private void initDataBase() { - if (persistenceCfg.getCheckpointThreads() > 1) - asyncRunner = new IgniteThreadPoolExecutor( - CHECKPOINT_RUNNER_THREAD_PREFIX, - cctx.igniteInstanceName(), - persistenceCfg.getCheckpointThreads(), - persistenceCfg.getCheckpointThreads(), - 30_000, - new LinkedBlockingQueue() - ); - } - - /** - * Try to register Metrics MBean. - * - * @throws IgniteCheckedException If failed. - */ - private void registrateMetricsMBean() throws IgniteCheckedException { - if (U.IGNITE_MBEANS_DISABLED) - return; - - try { - persistenceMetricsMbeanName = U.registerMBean( - cctx.kernalContext().config().getMBeanServer(), - cctx.kernalContext().igniteInstanceName(), - MBEAN_GROUP, - MBEAN_NAME, - persStoreMetrics, - DataStorageMetricsMXBean.class); - } - catch (Throwable e) { - throw new IgniteCheckedException("Failed to register " + MBEAN_NAME + " MBean.", e); - } - } - - /** - * Unregister metrics MBean. - */ - private void unRegistrateMetricsMBean() { - if (persistenceMetricsMbeanName == null) - return; - - assert !U.IGNITE_MBEANS_DISABLED; - - try { - cctx.kernalContext().config().getMBeanServer().unregisterMBean(persistenceMetricsMbeanName); - - persistenceMetricsMbeanName = null; - } - catch (Throwable e) { - U.error(log, "Failed to unregister " + MBEAN_NAME + " MBean.", e); - } + /** {@inheritDoc} */ + @Override protected void registerMetricsMBeans(IgniteConfiguration cfg) { + super.registerMetricsMBeans(cfg); + + registerMetricsMBean( + cctx.kernalContext().config(), + MBEAN_GROUP, + MBEAN_NAME, + persStoreMetrics, + DataStorageMetricsMXBean.class + ); } /** {@inheritDoc} */ @@ -926,8 +883,8 @@ protected WALPointer restoreBinaryMemory(Set cacheGrps) throws IgniteCh cacheGrps); if (tailWalPtr == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { - throw new StorageException("Restore wal pointer = " + tailWalPtr + ", while status.endPtr = " + - status.endPtr + ". Can't restore memory - critical part of WAL archive is missing."); + throw new StorageException("The memory cannot be restored. The critical part of WAL archive is missing " + + "[tailWalPtr=" + tailWalPtr + ", endPtr=" + status.endPtr + ']'); } nodeStart(tailWalPtr); @@ -1053,7 +1010,11 @@ public List> nodeStartedPointers() throws IgniteCheckedExce super.onKernalStop0(cancel); - unRegistrateMetricsMBean(); + unregisterMetricsMBean( + cctx.gridConfig(), + MBEAN_GROUP, + MBEAN_NAME + ); } /** {@inheritDoc} */ @@ -1226,8 +1187,9 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi plc = PageMemoryImpl.ThrottlingPolicy.valueOf(throttlingPolicyOverride.toUpperCase()); } catch (IllegalArgumentException e) { - log.error("Incorrect value of IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED property: " + - throttlingPolicyOverride + ". Default throttling policy " + plc + " will be used."); + log.error("Incorrect value of IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED property. " + + "The default throttling policy will be used [plc=" + throttlingPolicyOverride + + ", defaultPlc=" + plc + ']'); } } return plc; @@ -1239,8 +1201,8 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi if (!regCfg.isPersistenceEnabled()) super.checkRegionEvictionProperties(regCfg, dbCfg); else if (regCfg.getPageEvictionMode() != DataPageEvictionMode.DISABLED) { - U.warn(log, "Page eviction mode set for [" + regCfg.getName() + "] data will have no effect" + - " because the oldest pages are evicted automatically if Ignite persistence is enabled."); + U.warn(log, "Page eviction mode will have no effect because the oldest pages are evicted automatically " + + "if Ignite persistence is enabled: " + regCfg.getName()); } } @@ -2230,7 +2192,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC "[cpStatus=" + status + ", lastRead=" + lastReadPtr + "]"); log.info("Finished applying memory changes [changesApplied=" + applied + - ", time=" + (U.currentTimeMillis() - start) + "ms]"); + ", time=" + (U.currentTimeMillis() - start) + " ms]"); if (applied > 0) finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr); @@ -2309,7 +2271,8 @@ public void applyUpdatesOnRecovery( if (cacheCtx != null) applyUpdate(cacheCtx, dataEntry); else if (log != null) - log.warning("Cache (cacheId=" + cacheId + ") is not started, can't apply updates."); + log.warning("Cache is not started. Updates cannot be applied " + + "[cacheId=" + cacheId + ']'); } finally { checkpointReadUnlock(); @@ -2481,7 +2444,7 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th if (log.isInfoEnabled()) log.info("Finished applying WAL changes [updatesApplied=" + applied + - ", time=" + (U.currentTimeMillis() - start) + "ms]"); + ", time=" + (U.currentTimeMillis() - start) + " ms]"); } /** @@ -3198,7 +3161,7 @@ protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log) } finally { if (err == null && !(stopping && isCancelled)) - err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly"); + err = new IllegalStateException("Thread is terminated unexpectedly: " + name()); if (err instanceof OutOfMemoryError) cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err)); @@ -3829,7 +3792,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws } catch (IgniteException e) { U.error(log, "Failed to wait for snapshot operation initialization: " + - curr.snapshotOperation + "]", e); + curr.snapshotOperation, e); } } @@ -4494,9 +4457,9 @@ public void tryLock(long lockWaitTimeMillis) throws IgniteCheckedException { if (content == null) content = readContent(); - log.warning("Failed to acquire file lock (local nodeId:" + ctx.localNodeId() - + ", already locked by " + content + "), will try again in 1s: " - + file.getAbsolutePath()); + log.warning("Failed to acquire file lock. Will try again in 1s " + + "[nodeId=" + ctx.localNodeId() + ", holder=" + content + + ", path=" + file.getAbsolutePath() + ']'); } U.sleep(1000); @@ -4505,8 +4468,8 @@ public void tryLock(long lockWaitTimeMillis) throws IgniteCheckedException { if (content == null) content = readContent(); - failMsg = "Failed to acquire file lock during " + (lockWaitTimeMillis / 1000) + - " sec, (locked by " + content + "): " + file.getAbsolutePath(); + failMsg = "Failed to acquire file lock [holder=" + content + ", time=" + (lockWaitTimeMillis / 1000) + + " sec, path=" + file.getAbsolutePath() + ']'; } catch (Exception e) { throw new IgniteCheckedException(e); @@ -4792,7 +4755,7 @@ public WALRecord next(WALIterator it) throws IgniteCheckedException { return null; } - log.error("Catch error during restore state, throwsCRCError=" + throwsCRCError, e); + log.error("There is an error during restore state [throwsCRCError=" + throwsCRCError + ']', e); throw e; } @@ -4883,8 +4846,8 @@ public boolean needApplyBinaryUpdate() { * @return Flag indicates need throws CRC exception or not. */ @Override public boolean throwsCRCError() { - log.info("Throws CRC error check, needApplyBinaryUpdates=" + needApplyBinaryUpdates + - ", lastArchivedSegment=" + lastArchivedSegment + ", lastRead=" + lastRead); + log.info("Throws CRC error check [needApplyBinaryUpdates=" + needApplyBinaryUpdates + + ", lastArchivedSegment=" + lastArchivedSegment + ", lastRead=" + lastRead + ']'); if (needApplyBinaryUpdates) return true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 589b495bbf317..21bd45482cd14 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -93,6 +93,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** {@code True} to reuse memory on deactive. */ private final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); + /** */ + private static final String MBEAN_GROUP_NAME = "DataRegionMetrics"; + /** */ protected volatile Map dataRegionMap; @@ -140,44 +143,88 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap } /** - * Registers MBeans for all DataRegionMetrics configured in this instance. - */ - private void registerMetricsMBeans() { - if(U.IGNITE_MBEANS_DISABLED) + * @param cfg Ignite configuration. + * @param groupName Name of group. + * @param dataRegionName Metrics MBean name. + * @param impl Metrics implementation. + * @param clazz Metrics class type. + */ + protected void registerMetricsMBean( + IgniteConfiguration cfg, + String groupName, + String dataRegionName, + T impl, + Class clazz + ) { + if (U.IGNITE_MBEANS_DISABLED) return; - IgniteConfiguration cfg = cctx.gridConfig(); - - for (DataRegionMetrics memMetrics : memMetricsMap.values()) { - DataRegionConfiguration memPlcCfg = dataRegionMap.get(memMetrics.getName()).config(); - - registerMetricsMBean((DataRegionMetricsImpl)memMetrics, memPlcCfg, cfg); + try { + U.registerMBean( + cfg.getMBeanServer(), + cfg.getIgniteInstanceName(), + groupName, + dataRegionName, + impl, + clazz); + } + catch (Throwable e) { + U.error(log, "Failed to register MBean with name: " + dataRegionName, e); } } /** - * @param memMetrics Memory metrics. - * @param dataRegionCfg Data region configuration. * @param cfg Ignite configuration. + * @param groupName Name of group. + * @param name Name of MBean. */ - private void registerMetricsMBean( - DataRegionMetricsImpl memMetrics, - DataRegionConfiguration dataRegionCfg, - IgniteConfiguration cfg + protected void unregisterMetricsMBean( + IgniteConfiguration cfg, + String groupName, + String name ) { - assert !U.IGNITE_MBEANS_DISABLED; + if (U.IGNITE_MBEANS_DISABLED) + return; + + assert cfg != null; try { - U.registerMBean( - cfg.getMBeanServer(), - cfg.getIgniteInstanceName(), - "DataRegionMetrics", - dataRegionCfg.getName(), - new DataRegionMetricsMXBeanImpl(memMetrics, dataRegionCfg), - DataRegionMetricsMXBean.class); + cfg.getMBeanServer().unregisterMBean( + U.makeMBeanName( + cfg.getIgniteInstanceName(), + groupName, + name + )); + } + catch (InstanceNotFoundException ignored) { + // We tried to unregister a non-existing MBean, not a big deal. } catch (Throwable e) { - U.error(log, "Failed to register MBean for DataRegionMetrics with name: '" + memMetrics.getName() + "'", e); + U.error(log, "Failed to unregister MBean for memory metrics: " + name, e); + } + } + + /** + * Registers MBeans for all DataRegionMetrics configured in this instance. + * + * @param cfg Ignite configuration. + */ + protected void registerMetricsMBeans(IgniteConfiguration cfg) { + if (U.IGNITE_MBEANS_DISABLED) + return; + + assert cfg != null; + + for (DataRegionMetrics memMetrics : memMetricsMap.values()) { + DataRegionConfiguration memPlcCfg = dataRegionMap.get(memMetrics.getName()).config(); + + registerMetricsMBean( + cfg, + MBEAN_GROUP_NAME, + memPlcCfg.getName(), + new DataRegionMetricsMXBeanImpl((DataRegionMetricsImpl)memMetrics, memPlcCfg), + DataRegionMetricsMXBean.class + ); } } @@ -219,17 +266,6 @@ public int pageSize() { return pageSize; } - /** - * - */ - private void startMemoryPolicies() { - for (DataRegion memPlc : dataRegionMap.values()) { - memPlc.pageMemory().start(); - - memPlc.evictionTracker().start(); - } - } - /** * @param memCfg Database config. * @throws IgniteCheckedException If failed to initialize swap path. @@ -320,7 +356,7 @@ public void addDataRegion( dfltDataRegion = memPlc; else if (dataRegionName.equals(DFLT_DATA_REG_DEFAULT_NAME)) U.warn(log, "Data Region with name 'default' isn't used as a default. " + - "Please check Memory Policies configuration."); + "Please, check Data Region configuration."); } /** @@ -731,32 +767,6 @@ public ReuseList reuseList(String memPlcName) { onDeActivate(true); } - /** - * Unregister MBean. - * @param name Name of mbean. - */ - private void unregisterMBean(String name) { - if(U.IGNITE_MBEANS_DISABLED) - return; - - IgniteConfiguration cfg = cctx.gridConfig(); - - try { - cfg.getMBeanServer().unregisterMBean( - U.makeMBeanName( - cfg.getIgniteInstanceName(), - "DataRegionMetrics", name - )); - } - catch (InstanceNotFoundException ignored) { - // We tried to unregister a non-existing MBean, not a big deal. - } - catch (Throwable e) { - U.error(log, "Failed to unregister MBean for memory metrics: " + - name, e); - } - } - /** {@inheritDoc} */ @Override public boolean checkpointLockIsHeldByThread() { return true; @@ -1197,9 +1207,13 @@ private void startDataRegions(DataStorageConfiguration cfg) throws IgniteChecked assert cfg != null; - registerMetricsMBeans(); + registerMetricsMBeans(cctx.gridConfig()); - startMemoryPolicies(); + for (DataRegion memPlc : dataRegionMap.values()) { + memPlc.pageMemory().start(); + + memPlc.evictionTracker().start(); + } initPageMemoryDataStructures(cfg); @@ -1226,7 +1240,11 @@ private void onDeActivate(boolean shutdown) { memPlc.evictionTracker().stop(); - unregisterMBean(memPlc.memoryMetrics().getName()); + unregisterMetricsMBean( + cctx.gridConfig(), + MBEAN_GROUP_NAME, + memPlc.memoryMetrics().getName() + ); } dataRegionMap.clear(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java index 5c986eea23cff..7180d7bf16d10 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java @@ -76,7 +76,7 @@ private void tryToStartBackupClusterWhatShouldFail() { while (true) { String message = e.getMessage(); - if (message.contains("Failed to acquire file lock during")) + if (message.contains("Failed to acquire file lock [")) break; if (e.getCause() != null) From 94555f158013d94e3fd7e7f85c01c99e31a44611 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 1 Nov 2018 20:12:16 +0300 Subject: [PATCH 066/403] IGNITE-9420 Move logical recovery outside of PME - Fixes #5067. Signed-off-by: Alexey Goncharuk --- .../ignite/internal/GridKernalContext.java | 5 + .../internal/GridKernalContextImpl.java | 15 + .../apache/ignite/internal/IgniteKernal.java | 2 + .../eventstorage/GridEventStorageManager.java | 3 + .../pagemem/wal/record/DataRecord.java | 7 + .../wal/record/MemoryRecoveryRecord.java | 9 +- .../wal/record/MetastoreDataRecord.java | 8 +- .../MetaPageUpdatePartitionDataRecord.java | 3 +- .../cache/CacheAffinitySharedManager.java | 9 +- .../processors/cache/CacheGroupContext.java | 288 +++++- .../processors/cache/GridCacheContext.java | 44 +- .../cache/GridCacheEventManager.java | 4 + .../processors/cache/GridCacheProcessor.java | 324 ++++-- .../GridCacheSharedTtlCleanupManager.java | 15 +- .../GridDhtPartitionsExchangeFuture.java | 36 +- .../GridDhtPartitionTopologyImpl.java | 2 - .../cache/mvcc/MvccProcessorImpl.java | 16 +- .../DatabaseLifecycleListener.java | 23 +- .../GridCacheDatabaseSharedManager.java | 926 +++++++++--------- .../IgniteCacheDatabaseSharedManager.java | 98 +- .../persistence/file/AsyncFileIOFactory.java | 9 - .../file/EncryptedFileIOFactory.java | 7 - .../cache/persistence/file/FileIOFactory.java | 10 +- .../cache/persistence/file/FilePageStore.java | 7 +- .../file/RandomAccessFileIOFactory.java | 9 - .../MetastorageLifecycleListener.java | 4 +- .../partstate/PartitionRecoverState.java | 52 + .../snapshot/IgniteCacheSnapshotManager.java | 4 - .../wal/FileWriteAheadLogManager.java | 58 +- .../FsyncModeFileWriteAheadLogManager.java | 50 +- .../reader/StandaloneGridKernalContext.java | 5 + .../serializer/RecordDataV1Serializer.java | 4 +- .../IgniteClusterActivateDeactivateTest.java | 2 +- ...PdsBinaryMetadataOnClusterRestartTest.java | 6 +- .../IgnitePdsCorruptedIndexTest.java | 10 - .../IgnitePdsCorruptedStoreTest.java | 11 +- .../IgnitePdsPartitionFilesDestroyTest.java | 10 - .../IgnitePdsTaskCancelingTest.java | 5 - ...alModeChangeDuringRebalancingSelfTest.java | 5 - .../db/CheckpointBufferDeadlockTest.java | 5 - .../db/IgniteLogicalRecoveryTest.java | 577 +++++++++++ .../db/IgnitePdsDataRegionMetricsTest.java | 3 +- .../IgnitePdsDiskErrorsRecoveringTest.java | 71 +- ...IgniteNodeStoppedDuringDisableWALTest.java | 11 +- .../db/wal/IgniteWalFlushFailoverTest.java | 5 - ...lushMultiNodeFailoverAbstractSelfTest.java | 5 - .../wal/IgniteWalFormatFileFailoverTest.java | 9 +- .../wal/IgniteWalHistoryReservationsTest.java | 16 +- .../db/wal/IgniteWalRebalanceTest.java | 5 - .../wal/WalRecoveryTxLogicalRecordsTest.java | 6 +- .../pagemem/PagesWriteThrottleSmokeTest.java | 5 - .../AbstractNodeJoinTemplate.java | 4 +- .../wal/AbstractWalDeltaConsistencyTest.java | 2 + .../wal/ExplicitWalDeltaConsistencyTest.java | 2 + .../wal/memtracker/PageMemoryTracker.java | 13 +- .../PageMemoryTrackerPluginProvider.java | 19 +- .../StandaloneWalRecordsIteratorTest.java | 5 - ...tegrityWithPrimaryIndexCorruptionTest.java | 3 - .../hashmap/GridCacheTestContext.java | 1 + .../junits/GridAbstractTest.java | 7 +- .../AlignedBuffersDirectFileIOFactory.java | 5 - .../IgnitePdsWithIndexingCoreTestSuite.java | 3 + 62 files changed, 1959 insertions(+), 928 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java index 970b8e7150ac5..a43312cc8ac6a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java @@ -698,4 +698,9 @@ public interface GridKernalContext extends Iterable { * @return Default uncaught exception handler used by thread pools. */ public Thread.UncaughtExceptionHandler uncaughtExceptionHandler(); + + /** + * @return {@code True} if node is in recovery mode (before join to topology). + */ + public boolean recoveryMode(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java index f23e65042c0da..08090f2bff1f3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java @@ -415,6 +415,9 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable /** Failure processor. */ private FailureProcessor failureProc; + /** Recovery mode flag. Flag is set to {@code false} when discovery manager started. */ + private boolean recoveryMode = true; + /** * No-arg constructor is required by externalization. */ @@ -1180,6 +1183,18 @@ public Thread.UncaughtExceptionHandler uncaughtExceptionHandler() { return hnd; } + /** {@inheritDoc} */ + @Override public boolean recoveryMode() { + return recoveryMode; + } + + /** + * @param recoveryMode Recovery mode. + */ + public void recoveryMode(boolean recoveryMode) { + this.recoveryMode = recoveryMode; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(GridKernalContextImpl.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 250fbd78d3a54..b20023864ef07 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1048,6 +1048,8 @@ public void start( ctx.cache().context().database().notifyMetaStorageSubscribersOnReadyForRead(); ctx.cache().context().database().startMemoryRestore(ctx); + + ctx.recoveryMode(false); } catch (Throwable e) { U.error( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java index d4daab85c0ed7..92a2eefe0bcf3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java @@ -316,6 +316,9 @@ public void record(DiscoveryEvent evt, DiscoCache discoCache) { private void record0(Event evt, Object... params) { assert evt != null; + if (ctx.recoveryMode()) + return; + if (!enterBusy()) return; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java index 7a4d6b8793a21..d5ab53a7226c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java @@ -76,6 +76,13 @@ public DataRecord(List writeEntries, long timestamp) { this.writeEntries = writeEntries; } + /** + * @param writeEntries Write entries. + */ + public void setWriteEntries(List writeEntries) { + this.writeEntries = writeEntries; + } + /** * @return Collection of write entries. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java index 92658cc4ce788..5a48b340d0ca4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java @@ -20,16 +20,11 @@ import org.apache.ignite.internal.util.typedef.internal.S; /** - * Marker that we start memory recovering. - * - * @deprecated Previously, used to track node started\stopped states. But in fact only - * mark files created by method GridCacheDatabaseSharedManager#nodeStart(WALPointer) - * used. Should be removed in 3.0 release. + * Marker indicates that binary memory recovery has finished. */ -@Deprecated public class MemoryRecoveryRecord extends WALRecord { /** Create timestamp, millis */ - private long time; + private final long time; /** * Default constructor. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java index e269de2adc012..9e734244c7b84 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java @@ -18,13 +18,14 @@ package org.apache.ignite.internal.pagemem.wal.record; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.util.typedef.internal.S; import org.jetbrains.annotations.Nullable; /** * */ -public class MetastoreDataRecord extends WALRecord { +public class MetastoreDataRecord extends WALRecord implements WalRecordCacheGroupAware { /** */ private final String key; @@ -59,4 +60,9 @@ public String key() { @Override public String toString() { return S.toString(MetastoreDataRecord.class, this, "super", super.toString()); } + + /** {@inheritDoc} */ + @Override public int groupId() { + return MetaStorage.METASTORAGE_CACHE_ID; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java index e5bd343bcb5ee..28294a9f2cd5b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java @@ -55,7 +55,8 @@ public MetaPageUpdatePartitionDataRecord( long updateCntr, long globalRmvId, int partSize, - long cntrsPageId, byte state, + long cntrsPageId, + byte state, int allocatedIdxCandidate ) { super(grpId, pageId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 6e10c0009887f..1283696393a87 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache; -import javax.cache.CacheException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,6 +31,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; @@ -362,11 +362,7 @@ public Set waitGroups() { void onCacheGroupCreated(CacheGroupContext grp) { if (!grpHolders.containsKey(grp.groupId())) { cctx.io().addCacheGroupHandler(grp.groupId(), GridDhtAffinityAssignmentResponse.class, - new IgniteBiInClosure() { - @Override public void apply(UUID nodeId, GridDhtAffinityAssignmentResponse res) { - processAffinityAssignmentResponse(nodeId, res); - } - }); + (IgniteBiInClosure) this::processAffinityAssignmentResponse); } } @@ -1281,6 +1277,7 @@ private void forAllCacheGroups(boolean crd, IgniteInClosureX !grp.isLocal()) + .filter(grp -> !grp.isRecoveryMode()) .map(CacheGroupContext::affinity) .collect(Collectors.toList()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index a546a369f099e..b0d6c07acc27a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -22,8 +22,10 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.affinity.AffinityFunction; @@ -39,12 +41,18 @@ import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.query.continuous.CounterSkipContext; import org.apache.ignite.internal.processors.query.QueryUtils; @@ -78,13 +86,13 @@ public class CacheGroupContext { private final int grpId; /** Node ID cache group was received from. */ - private final UUID rcvdFrom; + private volatile UUID rcvdFrom; /** Flag indicating that this cache group is in a recovery mode due to partitions loss. */ private boolean needsRecovery; /** */ - private final AffinityTopologyVersion locStartVer; + private volatile AffinityTopologyVersion locStartVer; /** */ private final CacheConfiguration ccfg; @@ -93,7 +101,7 @@ public class CacheGroupContext { private final GridCacheSharedContext ctx; /** */ - private final boolean affNode; + private volatile boolean affNode; /** */ private final CacheType cacheType; @@ -117,16 +125,17 @@ public class CacheGroupContext { private final IgniteLogger log; /** */ - private GridAffinityAssignmentCache aff; + private volatile GridAffinityAssignmentCache aff; /** */ - private GridDhtPartitionTopologyImpl top; + private volatile GridDhtPartitionTopologyImpl top; /** */ - private IgniteCacheOffheapManager offheapMgr; + private volatile IgniteCacheOffheapManager offheapMgr; /** */ - private GridCachePreloader preldr; + private volatile GridCachePreloader preldr; + /** */ private final DataRegion dataRegion; @@ -160,6 +169,12 @@ public class CacheGroupContext { /** */ private volatile boolean globalWalEnabled; + /** Flag indicates that cache group is under recovering and not attached to topology. */ + private final AtomicBoolean recoveryMode; + + /** Flag indicates that all group partitions have restored their state from page memory / disk. */ + private volatile boolean partitionStatesRestored; + /** * @param ctx Context. * @param grpId Group ID. @@ -188,7 +203,8 @@ public class CacheGroupContext { ReuseList reuseList, AffinityTopologyVersion locStartVer, boolean persistenceEnabled, - boolean walEnabled + boolean walEnabled, + boolean recoveryMode ) { assert ccfg != null; assert dataRegion != null || !affNode; @@ -208,8 +224,7 @@ public class CacheGroupContext { this.globalWalEnabled = walEnabled; this.persistenceEnabled = persistenceEnabled; this.localWalEnabled = true; - - persistGlobalWalState(walEnabled); + this.recoveryMode = new AtomicBoolean(recoveryMode); ioPlc = cacheType.ioPolicy(); @@ -715,9 +730,11 @@ public boolean sharedGroup() { * */ public void onKernalStop() { - aff.cancelFutures(new IgniteCheckedException("Failed to wait for topology update, node is stopping.")); + if (!isRecoveryMode()) { + aff.cancelFutures(new IgniteCheckedException("Failed to wait for topology update, node is stopping.")); - preldr.onKernalStop(); + preldr.onKernalStop(); + } offheapMgr.onKernalStop(); } @@ -739,6 +756,11 @@ void stopCache(GridCacheContext cctx, boolean destroy) { * */ void stopGroup() { + offheapMgr.stop(); + + if (isRecoveryMode()) + return; + IgniteCheckedException err = new IgniteCheckedException("Failed to wait for topology update, cache (or node) is stopping."); @@ -748,11 +770,204 @@ void stopGroup() { preldr.onKernalStop(); - offheapMgr.stop(); - ctx.io().removeCacheGroupHandlers(grpId); } + /** + * Finishes recovery for current cache group. + * Attaches topology version and initializes I/O. + * + * @param startVer Cache group start version. + * @param originalReceivedFrom UUID of node that was first who initiated cache group creating. + * This is needed to decide should node calculate affinity locally or fetch from other nodes. + * @param affinityNode Flag indicates, is local node affinity node or not. This may be calculated only after node joined to topology. + * @throws IgniteCheckedException If failed. + */ + public void finishRecovery( + AffinityTopologyVersion startVer, + UUID originalReceivedFrom, + boolean affinityNode + ) throws IgniteCheckedException { + if (recoveryMode.compareAndSet(true, false)) { + affNode = affinityNode; + + rcvdFrom = originalReceivedFrom; + + locStartVer = startVer; + + persistGlobalWalState(globalWalEnabled); + + initializeIO(); + + ctx.affinity().onCacheGroupCreated(this); + } + } + + /** + * Pre-create partitions that resides in page memory or WAL and restores their state. + */ + public long restorePartitionStates(Map partitionRecoveryStates) throws IgniteCheckedException { + if (isLocal() || !affinityNode() || !dataRegion().config().isPersistenceEnabled()) + return 0; + + if (partitionStatesRestored) + return 0; + + long processed = 0; + + PageMemoryEx pageMem = (PageMemoryEx)dataRegion().pageMemory(); + + for (int p = 0; p < affinity().partitions(); p++) { + PartitionRecoverState recoverState = partitionRecoveryStates.get(new GroupPartitionId(grpId, p)); + + if (ctx.pageStore().exists(grpId, p)) { + ctx.pageStore().ensure(grpId, p); + + if (ctx.pageStore().pages(grpId, p) <= 1) { + if (log.isDebugEnabled()) + log.debug("Skipping partition on recovery (pages less than 1) " + + "[grp=" + cacheOrGroupName() + ", p=" + p + "]"); + + continue; + } + + if (log.isDebugEnabled()) + log.debug("Creating partition on recovery (exists in page store) " + + "[grp=" + cacheOrGroupName() + ", p=" + p + "]"); + + processed++; + + GridDhtLocalPartition part = topology().forceCreatePartition(p); + + offheap().onPartitionInitialCounterUpdated(p, 0); + + ctx.database().checkpointReadLock(); + + try { + long partMetaId = pageMem.partitionMetaPageId(grpId, p); + long partMetaPage = pageMem.acquirePage(grpId, partMetaId); + + try { + long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage); + + boolean changed = false; + + try { + PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr); + + if (recoverState != null) { + io.setPartitionState(pageAddr, (byte) recoverState.stateId()); + + changed = updateState(part, recoverState.stateId()); + + if (recoverState.stateId() == GridDhtPartitionState.OWNING.ordinal() + || (recoverState.stateId() == GridDhtPartitionState.MOVING.ordinal() + && part.initialUpdateCounter() < recoverState.updateCounter())) { + part.initialUpdateCounter(recoverState.updateCounter()); + + changed = true; + } + + if (log.isInfoEnabled()) + log.warning("Restored partition state (from WAL) " + + "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + + ", updCntr=" + part.initialUpdateCounter() + "]"); + } + else { + int stateId = (int) io.getPartitionState(pageAddr); + + changed = updateState(part, stateId); + + if (log.isDebugEnabled()) + log.debug("Restored partition state (from page memory) " + + "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + + ", updCntr=" + part.initialUpdateCounter() + ", stateId=" + stateId + "]"); + } + } + finally { + pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed); + } + } + finally { + pageMem.releasePage(grpId, partMetaId, partMetaPage); + } + } + finally { + ctx.database().checkpointReadUnlock(); + } + } + else if (recoverState != null) { + GridDhtLocalPartition part = topology().forceCreatePartition(p); + + offheap().onPartitionInitialCounterUpdated(p, recoverState.updateCounter()); + + updateState(part, recoverState.stateId()); + + processed++; + + if (log.isDebugEnabled()) + log.debug("Restored partition state (from WAL) " + + "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + + ", updCntr=" + part.initialUpdateCounter() + "]"); + } + else { + if (log.isDebugEnabled()) + log.debug("Skipping partition on recovery (no page store OR wal state) " + + "[grp=" + cacheOrGroupName() + ", p=" + p + "]"); + } + } + + partitionStatesRestored = true; + + return processed; + } + + /** + * @param part Partition to restore state for. + * @param stateId State enum ordinal. + * @return Updated flag. + */ + private boolean updateState(GridDhtLocalPartition part, int stateId) { + if (stateId != -1) { + GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId); + + assert state != null; + + part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state); + + return true; + } + + return false; + } + + /** + * @return {@code True} if current cache group is in recovery mode. + */ + public boolean isRecoveryMode() { + return recoveryMode.get(); + } + + /** + * Initializes affinity and rebalance I/O handlers. + */ + private void initializeIO() throws IgniteCheckedException { + assert !recoveryMode.get() : "Couldn't initialize I/O handlers, recovery mode is on for group " + this; + + if (ccfg.getCacheMode() != LOCAL) { + if (!ctx.kernalContext().clientNode()) { + ctx.io().addCacheGroupHandler(groupId(), GridDhtAffinityAssignmentRequest.class, + (IgniteBiInClosure) this::processAffinityAssignmentRequest); + } + + preldr = new GridDhtPreloader(this); + + preldr.start(); + } + else + preldr = new GridCachePreloaderAdapter(this); + } + /** * @return IDs of caches in this group. */ @@ -777,7 +992,7 @@ public List caches() { /** * @return {@code True} if group contains caches. */ - boolean hasCaches() { + public boolean hasCaches() { List caches = this.caches; return !caches.isEmpty(); @@ -904,39 +1119,25 @@ public void start() throws IgniteCheckedException { ccfg.getCacheMode() == LOCAL, persistenceEnabled()); - if (ccfg.getCacheMode() != LOCAL) { + if (ccfg.getCacheMode() != LOCAL) top = new GridDhtPartitionTopologyImpl(ctx, this); - if (!ctx.kernalContext().clientNode()) { - ctx.io().addCacheGroupHandler(groupId(), GridDhtAffinityAssignmentRequest.class, - new IgniteBiInClosure() { - @Override public void apply(UUID nodeId, GridDhtAffinityAssignmentRequest msg) { - processAffinityAssignmentRequest(nodeId, msg); - } - }); - } - - preldr = new GridDhtPreloader(this); - - preldr.start(); + try { + offheapMgr = persistenceEnabled + ? new GridCacheOffheapManager() + : new IgniteCacheOffheapManagerImpl(); } - else - preldr = new GridCachePreloaderAdapter(this); - - if (persistenceEnabled()) { - try { - offheapMgr = new GridCacheOffheapManager(); - } - catch (Exception e) { - throw new IgniteCheckedException("Failed to initialize offheap manager", e); - } + catch (Exception e) { + throw new IgniteCheckedException("Failed to initialize offheap manager", e); } - else - offheapMgr = new IgniteCacheOffheapManagerImpl(); offheapMgr.start(ctx, this); - ctx.affinity().onCacheGroupCreated(this); + if (!isRecoveryMode()) { + initializeIO(); + + ctx.affinity().onCacheGroupCreated(this); + } } /** @@ -950,8 +1151,7 @@ public boolean persistenceEnabled() { * @param nodeId Node ID. * @param req Request. */ - private void processAffinityAssignmentRequest(final UUID nodeId, - final GridDhtAffinityAssignmentRequest req) { + private void processAffinityAssignmentRequest(UUID nodeId, GridDhtAffinityAssignmentRequest req) { if (log.isDebugEnabled()) log.debug("Processing affinity assignment request [node=" + nodeId + ", req=" + req + ']'); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 7eea905966b63..ff430b6eeea91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -226,7 +226,7 @@ public class GridCacheContext implements Externalizable { private CacheWeakQueryIteratorsHolder> itHolder; /** Affinity node. */ - private boolean affNode; + private volatile boolean affNode; /** Conflict resolver. */ private CacheVersionConflictResolver conflictRslvr; @@ -238,7 +238,7 @@ public class GridCacheContext implements Externalizable { private CountDownLatch startLatch = new CountDownLatch(1); /** Topology version when cache was started on local node. */ - private AffinityTopologyVersion locStartTopVer; + private volatile AffinityTopologyVersion locStartTopVer; /** Dynamic cache deployment ID. */ private IgniteUuid dynamicDeploymentId; @@ -271,7 +271,10 @@ public class GridCacheContext implements Externalizable { private boolean readFromBackup = CacheConfiguration.DFLT_READ_FROM_BACKUP; /** Local node's MAC address. */ - private String locMacs; + private volatile String locMacs; + + /** Recovery mode flag. */ + private volatile boolean recoveryMode; /** * Empty constructor required for {@link Externalizable}. @@ -311,6 +314,7 @@ public GridCacheContext( AffinityTopologyVersion locStartTopVer, boolean affNode, boolean updatesAllowed, + boolean recoveryMode, /* * Managers in starting order! @@ -395,11 +399,45 @@ public GridCacheContext( readFromBackup = cacheCfg.isReadFromBackup(); + this.recoveryMode = recoveryMode; + + assert kernalContext().recoveryMode() == recoveryMode; + + if (!recoveryMode) { + locMacs = localNode().attribute(ATTR_MACS); + + assert locMacs != null; + } + } + + /** + * Called when cache was restored during recovery and node has joined to topology. + * + * @param topVer Cache topology join version. + * @param statisticsEnabled Flag indicates is statistics enabled or not for that cache. + * Value may be changed after node joined to topology. + */ + public void finishRecovery(AffinityTopologyVersion topVer, boolean statisticsEnabled) { + assert recoveryMode : this; + + recoveryMode = false; + + locStartTopVer = topVer; + locMacs = localNode().attribute(ATTR_MACS); + this.statisticsEnabled = statisticsEnabled; + assert locMacs != null; } + /** + * @return {@code True} if cache is in recovery mode. + */ + public boolean isRecoveryMode() { + return recoveryMode; + } + /** * @return Cache group ID. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java index 3c5cf1e944ad1..ef350160eb11a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java @@ -372,6 +372,10 @@ public void addEvent( public boolean isRecordable(int type) { GridCacheContext cctx0 = cctx; + // Event recording is impossible in recovery mode. + if (cctx0 != null && cctx0.kernalContext().recoveryMode()) + return false; + return cctx0 != null && cctx0.userCache() && cctx0.gridEvents().isRecordable(type) && !cctx0.config().isEventsDisabled(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index ca0da8807c2c0..27947dc691e4c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -100,6 +100,7 @@ import org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache; import org.apache.ignite.internal.processors.cache.mvcc.MvccCachingManager; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; @@ -107,7 +108,6 @@ import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage; -import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; @@ -202,7 +202,7 @@ * Cache processor. */ @SuppressWarnings({"unchecked", "TypeMayBeWeakened", "deprecation"}) -public class GridCacheProcessor extends GridProcessorAdapter implements MetastorageLifecycleListener { +public class GridCacheProcessor extends GridProcessorAdapter { /** Template of message of conflicts during configuration merge*/ private static final String MERGE_OF_CONFIG_CONFLICTS_MESSAGE = "Conflicts during configuration merge for cache '%s' : \n%s"; @@ -277,6 +277,9 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor /** Protector of initialization of specific value. */ private final InitializationProtector initializationProtector = new InitializationProtector(); + /** Cache recovery lifecycle state and actions. */ + private final CacheRecoveryLifecycle recovery = new CacheRecoveryLifecycle(); + /** * @param ctx Kernal context. */ @@ -743,41 +746,34 @@ private void cleanup(CacheConfiguration cfg, @Nullable Object rsrc, boolean near } } - /** {@inheritDoc} */ - @Override public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException { - startCachesOnStart(); - } - - /** {@inheritDoc} */ - @Override public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException { - } - /** * @throws IgniteCheckedException If failed. */ - private void startCachesOnStart() throws IgniteCheckedException { - if (!ctx.isDaemon()) { - Map caches = new HashMap<>(); + private void restoreCacheConfigurations() throws IgniteCheckedException { + if (ctx.isDaemon()) + return; - Map templates = new HashMap<>(); + Map caches = new HashMap<>(); - addCacheOnJoinFromConfig(caches, templates); + Map templates = new HashMap<>(); - CacheJoinNodeDiscoveryData discoData = new CacheJoinNodeDiscoveryData( + addCacheOnJoinFromConfig(caches, templates); + + CacheJoinNodeDiscoveryData discoData = new CacheJoinNodeDiscoveryData( IgniteUuid.randomUuid(), caches, templates, startAllCachesOnClientStart() - ); + ); - cachesInfo.onStart(discoData); - } + cachesInfo.onStart(discoData); } /** {@inheritDoc} */ @SuppressWarnings({"unchecked"}) @Override public void start() throws IgniteCheckedException { - ctx.internalSubscriptionProcessor().registerMetastorageListener(this); + ctx.internalSubscriptionProcessor().registerMetastorageListener(recovery); + ctx.internalSubscriptionProcessor().registerDatabaseListener(recovery); cachesInfo = new ClusterCachesInfo(ctx); @@ -803,7 +799,7 @@ private void startCachesOnStart() throws IgniteCheckedException { mgr.start(sharedCtx); if (!ctx.isDaemon() && (!CU.isPersistenceEnabled(ctx.config())) || ctx.config().isClientMode()) - startCachesOnStart(); + restoreCacheConfigurations(); if (log.isDebugEnabled()) log.debug("Started cache processor."); @@ -1462,7 +1458,7 @@ private void onKernalStop(GridCacheAdapter cache, boolean cancel) { cache.onKernalStop(); - if (ctx.events().isRecordable(EventType.EVT_CACHE_STOPPED)) + if (!ctx.isRecoveryMode() && ctx.events().isRecordable(EventType.EVT_CACHE_STOPPED)) ctx.events().addEvent(EventType.EVT_CACHE_STOPPED); } @@ -1480,7 +1476,8 @@ private void onKernalStop(GridCacheAdapter cache, boolean cancel) { * @return Cache context. * @throws IgniteCheckedException If failed to create cache. */ - private GridCacheContext createCache(CacheConfiguration cfg, + private GridCacheContext createCacheContext( + CacheConfiguration cfg, CacheGroupContext grp, @Nullable CachePluginManager pluginMgr, DynamicCacheDescriptor desc, @@ -1488,8 +1485,9 @@ private GridCacheContext createCache(CacheConfiguration cfg, CacheObjectContext cacheObjCtx, boolean affNode, boolean updatesAllowed, - boolean disabledAfterStart) - throws IgniteCheckedException { + boolean disabledAfterStart, + boolean recoveryMode + ) throws IgniteCheckedException { assert cfg != null; if (cfg.getCacheStoreFactory() instanceof GridCacheLoaderWriterStoreFactory) { @@ -1563,6 +1561,7 @@ private GridCacheContext createCache(CacheConfiguration cfg, locStartTopVer, affNode, updatesAllowed, + recoveryMode, /* * Managers in starting order! * =========================== @@ -1698,6 +1697,7 @@ private GridCacheContext createCache(CacheConfiguration cfg, locStartTopVer, affNode, true, + recoveryMode, /* * Managers in starting order! * =========================== @@ -2132,12 +2132,16 @@ private void prepareStartCaches( cacheStartFailHandler.handle( startCacheInfo, cacheInfo -> { - ctx.query().onCacheStart( - cacheContexts.get(cacheInfo), - cacheInfo.getCacheDescriptor().schema() != null - ? cacheInfo.getCacheDescriptor().schema() - : new QuerySchema() - ); + GridCacheContext cctx = cacheContexts.get(cacheInfo); + + if (!cctx.isRecoveryMode()) { + ctx.query().onCacheStart( + cctx, + cacheInfo.getCacheDescriptor().schema() != null + ? cacheInfo.getCacheDescriptor().schema() + : new QuerySchema() + ); + } context().exchange().exchangerUpdateHeartbeat(); } @@ -2152,7 +2156,12 @@ private void prepareStartCaches( cacheStartFailHandler.handle( cacheCtxEntry.getKey(), cacheInfo -> { - onCacheStarted(cacheCtxEntry.getValue()); + GridCacheContext cacheContext = cacheCtxEntry.getValue(); + + if (cacheContext.isRecoveryMode()) + finishRecovery(cacheInfo.getExchangeTopVer(), cacheContext); + else + onCacheStarted(cacheCtxEntry.getValue()); context().exchange().exchangerUpdateHeartbeat(); } @@ -2170,7 +2179,7 @@ private void prepareStartCaches( * change state of proxies to restarting * @throws IgniteCheckedException If failed. */ - void prepareCacheStart( + public void prepareCacheStart( CacheConfiguration startCfg, DynamicCacheDescriptor desc, @Nullable NearCacheConfiguration reqNearCfg, @@ -2181,7 +2190,10 @@ void prepareCacheStart( ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); - onCacheStarted(cacheCtx); + if (cacheCtx.isRecoveryMode()) + finishRecovery(exchTopVer, cacheCtx); + else + onCacheStarted(cacheCtx); } /** @@ -2203,6 +2215,24 @@ private GridCacheContext prepareCacheContext( AffinityTopologyVersion exchTopVer, boolean disabledAfterStart ) throws IgniteCheckedException { + if (caches.containsKey(startCfg.getName())) { + GridCacheAdapter existingCache = caches.get(startCfg.getName()); + + GridCacheContext cctx = existingCache.context(); + + assert cctx.isRecoveryMode(); + + QuerySchema localSchema = recovery.querySchemas.get(desc.cacheId()); + + QuerySchemaPatch localSchemaPatch = localSchema.makePatch(desc.schema().entities()); + + // Cache schema is changed after restart, workaround is stop existing cache and start new. + if (!localSchemaPatch.isEmpty() || localSchemaPatch.hasConflicts()) + stopCacheSafely(cctx); + else + return existingCache.context(); + } + assert !caches.containsKey(startCfg.getName()) : startCfg.getName(); CacheConfiguration ccfg = new CacheConfiguration(startCfg); @@ -2211,9 +2241,9 @@ private GridCacheContext prepareCacheContext( boolean affNode = checkForAffinityNode(desc, reqNearCfg, ccfg); - CacheGroupContext grp = prepareCacheGroup(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName()); + CacheGroupContext grp = getOrCreateCacheGroupContext(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName(), false); - GridCacheContext cacheCtx = createCache(ccfg, + GridCacheContext cacheCtx = createCacheContext(ccfg, grp, null, desc, @@ -2221,7 +2251,8 @@ private GridCacheContext prepareCacheContext( cacheObjCtx, affNode, true, - disabledAfterStart + disabledAfterStart, + false ); initCacheContext(cacheCtx, ccfg, desc.deploymentId()); @@ -2229,6 +2260,74 @@ private GridCacheContext prepareCacheContext( return cacheCtx; } + /** + * Stops cache under checkpoint lock. + * @param cctx Cache context. + */ + private void stopCacheSafely(GridCacheContext cctx) { + sharedCtx.database().checkpointReadLock(); + + try { + prepareCacheStop(cctx.name(), false); + + if (!cctx.group().hasCaches()) + stopCacheGroup(cctx.group().groupId()); + } + finally { + sharedCtx.database().checkpointReadUnlock(); + } + + } + + /** + * Finishes recovery for given cache context. + * + * @param cacheStartVer Cache join to topology version. + * @param cacheContext Cache context. + * @throws IgniteCheckedException If failed. + */ + private void finishRecovery(AffinityTopologyVersion cacheStartVer, GridCacheContext cacheContext) throws IgniteCheckedException { + CacheGroupContext groupContext = cacheContext.group(); + + // Take cluster-wide cache descriptor and try to update local cache and cache group parameters. + DynamicCacheDescriptor updatedDescriptor = cacheDescriptor(cacheContext.cacheId()); + + groupContext.finishRecovery( + cacheStartVer, + updatedDescriptor.receivedFrom(), + isLocalAffinity(updatedDescriptor.cacheConfiguration()) + ); + + cacheContext.finishRecovery(cacheStartVer, updatedDescriptor.cacheConfiguration().isStatisticsEnabled()); + + onKernalStart(cacheContext.cache()); + + if (log.isInfoEnabled()) + log.info("Finished recovery for cache [cache=" + cacheContext.name() + + ", grp=" + groupContext.cacheOrGroupName() + ", startVer=" + cacheStartVer + "]"); + } + + /** + * Stops all caches and groups, that was recovered, but not activated on node join. + * Such caches can remain only if it was filtered by node filter on current node. + * It's impossible to check whether current node is affinity node for given cache before join to topology. + */ + public void shutdownNotFinishedRecoveryCaches() { + for (GridCacheAdapter cacheAdapter : caches.values()) { + GridCacheContext cacheContext = cacheAdapter.context(); + + if (cacheContext.isLocal()) + continue; + + if (cacheContext.isRecoveryMode()) { + assert !isLocalAffinity(cacheContext.config()) + : "Cache " + cacheAdapter.context() + " is still in recovery mode after start, but not activated."; + + stopCacheSafely(cacheContext); + } + } + } + /** * Check for affinity node and customize near configuration if needed. * @@ -2282,12 +2381,13 @@ public void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throw * @return Prepared cache group context. * @throws IgniteCheckedException if failed. */ - private CacheGroupContext prepareCacheGroup( + private CacheGroupContext getOrCreateCacheGroupContext( DynamicCacheDescriptor desc, AffinityTopologyVersion exchTopVer, CacheObjectContext cacheObjCtx, boolean affNode, - String grpName + String grpName, + boolean recoveryMode ) throws IgniteCheckedException { if (grpName != null) { return initializationProtector.protect( @@ -2298,7 +2398,8 @@ private CacheGroupContext prepareCacheGroup( desc.cacheType(), affNode, cacheObjCtx, - exchTopVer + exchTopVer, + recoveryMode ) ); } @@ -2307,7 +2408,8 @@ private CacheGroupContext prepareCacheGroup( desc.cacheType(), affNode, cacheObjCtx, - exchTopVer + exchTopVer, + recoveryMode ); } @@ -2399,6 +2501,67 @@ private void onCacheStarted(GridCacheContext cacheCtx) throws IgniteCheckedExcep onKernalStart(cache); } + /** + * @param desc Cache descriptor. + * @throws IgniteCheckedException If failed. + */ + private GridCacheContext startCacheInRecoveryMode( + DynamicCacheDescriptor desc + ) throws IgniteCheckedException { + CacheConfiguration cfg = desc.cacheConfiguration(); + + CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(cfg); + + preparePageStore(desc, true); + + CacheGroupContext grp = getOrCreateCacheGroupContext( + desc, + AffinityTopologyVersion.NONE, + cacheObjCtx, + true, + cfg.getGroupName(), + true + ); + + GridCacheContext cacheCtx = createCacheContext(cfg, + grp, + null, + desc, + AffinityTopologyVersion.NONE, + cacheObjCtx, + true, + true, + false, + true + ); + + initCacheContext(cacheCtx, cfg, desc.deploymentId()); + + cacheCtx.onStarted(); + + String dataRegion = cfg.getDataRegionName(); + + if (dataRegion == null && ctx.config().getDataStorageConfiguration() != null) + dataRegion = ctx.config().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName(); + + grp.onCacheStarted(cacheCtx); + + ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); + + if (log.isInfoEnabled()) { + log.info("Started cache in recovery mode [name=" + cfg.getName() + + ", id=" + cacheCtx.cacheId() + + (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") + + ", dataRegionName=" + dataRegion + + ", mode=" + cfg.getCacheMode() + + ", atomicity=" + cfg.getAtomicityMode() + + ", backups=" + cfg.getBackups() + + ", mvcc=" + cacheCtx.mvccEnabled() + ']'); + } + + return cacheCtx; + } + /** * @param grpName Group name. * @return Found group or null. @@ -2449,8 +2612,9 @@ private CacheGroupContext startCacheGroup( CacheType cacheType, boolean affNode, CacheObjectContext cacheObjCtx, - AffinityTopologyVersion exchTopVer) - throws IgniteCheckedException { + AffinityTopologyVersion exchTopVer, + boolean recoveryMode + ) throws IgniteCheckedException { CacheConfiguration cfg = new CacheConfiguration(desc.config()); String memPlcName = cfg.getDataRegionName(); @@ -2459,7 +2623,7 @@ private CacheGroupContext startCacheGroup( FreeList freeList = sharedCtx.database().freeList(memPlcName); ReuseList reuseList = sharedCtx.database().reuseList(memPlcName); - boolean persistenceEnabled = sharedCtx.localNode().isClient() ? desc.persistenceEnabled() : + boolean persistenceEnabled = recoveryMode || sharedCtx.localNode().isClient() ? desc.persistenceEnabled() : dataRegion != null && dataRegion.config().isPersistenceEnabled(); CacheGroupContext grp = new CacheGroupContext(sharedCtx, @@ -2474,7 +2638,8 @@ private CacheGroupContext startCacheGroup( reuseList, exchTopVer, persistenceEnabled, - desc.walEnabled() + desc.walEnabled(), + recoveryMode ); for (Object obj : grp.configuredUserObjects()) @@ -2567,7 +2732,7 @@ private void stopGateway(DynamicCacheChangeRequest req) { * @param destroy Cache data destroy flag. Setting to true will remove all cache data. * @return Stopped cache context. */ - private GridCacheContext prepareCacheStop(String cacheName, boolean destroy) { + public GridCacheContext prepareCacheStop(String cacheName, boolean destroy) { assert sharedCtx.database().checkpointLockIsHeldByThread(); GridCacheAdapter cache = caches.remove(cacheName); @@ -2669,17 +2834,7 @@ private void closeCache(GridCacheContext cctx) { jCacheProxies.remove(cctx.name()); - sharedCtx.database().checkpointReadLock(); - - try { - prepareCacheStop(cctx.name(), false); - } - finally { - sharedCtx.database().checkpointReadUnlock(); - } - - if (!cctx.group().hasCaches()) - stopCacheGroup(cctx.group().groupId()); + stopCacheSafely(cctx); } finally { sharedCtx.io().writeUnlock(); @@ -4572,6 +4727,26 @@ public Map cacheDescriptors() { return cachesInfo.registeredCaches(); } + /** + * @return Collection of persistent cache descriptors. + */ + public Collection persistentCaches() { + return cachesInfo.registeredCaches().values() + .stream() + .filter(desc -> isPersistentCache(desc.cacheConfiguration(), ctx.config().getDataStorageConfiguration())) + .collect(Collectors.toList()); + } + + /** + * @return Collection of persistent cache group descriptors. + */ + public Collection persistentGroups() { + return cachesInfo.registeredCacheGroups().values() + .stream() + .filter(CacheGroupDescriptor::persistenceEnabled) + .collect(Collectors.toList()); + } + /** * @return Cache group descriptors. */ @@ -5255,6 +5430,39 @@ public T clone(final T obj) throws IgniteCheckedException { }); } + /** + * Recovery lifecycle for caches. + */ + private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, DatabaseLifecycleListener { + /** Set of QuerySchema's saved on recovery. It's needed if cache query schema has changed after node joined to topology.*/ + private final Map querySchemas = new ConcurrentHashMap<>(); + + /** {@inheritDoc} */ + @Override public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException { + restoreCacheConfigurations(); + } + + /** {@inheritDoc} */ + @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + for (DynamicCacheDescriptor cacheDescriptor : persistentCaches()) + preparePageStore(cacheDescriptor, true); + } + + /** {@inheritDoc} */ + @Override public void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + for (DynamicCacheDescriptor cacheDescriptor : persistentCaches()) { + // Skip MVCC caches. + // TODO: https://issues.apache.org/jira/browse/IGNITE-10052 + if (cacheDescriptor.cacheConfiguration().getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + continue; + + startCacheInRecoveryMode(cacheDescriptor); + + querySchemas.put(cacheDescriptor.cacheId(), cacheDescriptor.schema().copy()); + } + } + } + /** * Handle of fail during cache start. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java index 7a543546eb700..ad2342b15110f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java @@ -133,14 +133,17 @@ private class CleanupWorker extends GridWorker { while (!isCancelled()) { boolean expiredRemains = false; - for (GridCacheTtlManager mgr : mgrs) { - updateHeartbeat(); + // TTL cleanup is allowed only when node joined to topology. + if (!cctx.kernalContext().recoveryMode()) { + for (GridCacheTtlManager mgr : mgrs) { + updateHeartbeat(); - if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT)) - expiredRemains = true; + if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT)) + expiredRemains = true; - if (isCancelled()) - return; + if (isCancelled()) + return; + } } updateHeartbeat(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 9933e11994d07..0636e90ff962d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -870,10 +870,15 @@ else if (msg instanceof WalStateAbstractMessage) * @throws IgniteCheckedException If failed. */ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedException { - if (!isLocalNodeInBaseline()) { + if (!cctx.kernalContext().clientNode() && !isLocalNodeInBaseline()) { cctx.exchange().exchangerBlockingSectionBegin(); try { + // Stop all recovered caches and groups. + cctx.cache().onKernalStopCaches(true); + + cctx.cache().stopCaches(true); + cctx.database().cleanupRestoredCaches(); for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) { @@ -901,12 +906,12 @@ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedExce cctx.exchange().exchangerBlockingSectionEnd(); } - if (!cctx.kernalContext().clientNode()) - cctx.database().onDoneRestoreBinaryMemory(); - IgniteInternalFuture cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), exchActions == null ? null : exchActions.localJoinContext()); + if (!cctx.kernalContext().clientNode()) + cctx.cache().shutdownNotFinishedRecoveryCaches(); + ensureClientCachesStarted(); return cachesRegistrationFut; @@ -1066,15 +1071,15 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) { cctx.exchange().exchangerBlockingSectionEnd(); } - if (!cctx.kernalContext().clientNode()) - cctx.database().onDoneRestoreBinaryMemory(); - assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started."; cctx.exchange().exchangerBlockingSectionBegin(); try { registerCachesFuture = cctx.affinity().onCacheChangeRequest(this, crd, exchActions); + + if (!cctx.kernalContext().clientNode()) + cctx.cache().shutdownNotFinishedRecoveryCaches(); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -1407,16 +1412,15 @@ private void distributedExchange() throws IgniteCheckedException { } } - /* It is necessary to run database callback before all topology callbacks. - In case of persistent store is enabled we first restore partitions presented on disk. - We need to guarantee that there are no partition state changes logged to WAL before this callback - to make sure that we correctly restored last actual states. */ - boolean restored; - cctx.exchange().exchangerBlockingSectionBegin(); try { - restored = cctx.database().beforeExchange(this); + /* It is necessary to run database callback before all topology callbacks. + In case of persistent store is enabled we first restore partitions presented on disk. + We need to guarantee that there are no partition state changes logged to WAL before this callback + to make sure that we correctly restored last actual states. */ + + cctx.database().beforeExchange(this); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -1443,11 +1447,11 @@ private void distributedExchange() throws IgniteCheckedException { } // After all partitions have been restored and pre-created it's safe to make first checkpoint. - if (restored) { + if (localJoinExchange() || activateCluster()) { cctx.exchange().exchangerBlockingSectionBegin(); try { - cctx.database().onStateRestored(); + cctx.database().onStateRestored(initialVersion()); } finally { cctx.exchange().exchangerBlockingSectionEnd(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 683a140c5d978..5f5155cd7626c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -885,8 +885,6 @@ private GridDhtLocalPartition getOrCreatePartition(int p) { locParts.set(p, part); - ctx.pageStore().onPartitionCreated(grp.groupId(), p); - return part; } finally { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java index 7a4fa39aaea6e..7118d039b7c47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java @@ -319,18 +319,13 @@ public MvccProcessorImpl(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void afterInitialise(IgniteCacheDatabaseSharedManager mgr) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + @Override public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + // In case of blt changed we should re-init TX_LOG cache. txLogPageStoreInit(mgr); } /** {@inheritDoc} */ - @Override public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { - // In case of blt changed we should re-init TX_LOG cache. + @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { txLogPageStoreInit(mgr); } @@ -345,11 +340,6 @@ private void txLogPageStoreInit(IgniteCacheDatabaseSharedManager mgr) throws Ign TX_LOG_CACHE_NAME, mgr.dataRegion(TX_LOG_CACHE_NAME).memoryMetrics()); } - /** {@inheritDoc} */ - @Override public void afterMemoryRestore(IgniteCacheDatabaseSharedManager mgr) { - // No-op. - } - /** {@inheritDoc} */ @Override public void onDiscoveryEvent(int evtType, Collection nodes, long topVer, @Nullable DiscoveryCustomMessage customMsg) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java index ae65c77cc6e96..1f7ba84a05d02 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java @@ -24,36 +24,43 @@ */ public interface DatabaseLifecycleListener { /** - * @param mgr Database shared manager. + * Callback executed when data regions become to start-up. * + * @param mgr Database shared manager. + * @throws IgniteCheckedException If failed. */ - void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + default void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; /** + * Callback executed right before node become perform binary recovery. + * * @param mgr Database shared manager. * @throws IgniteCheckedException If failed. */ - public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + default void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; /** + * Callback executed when binary memory has fully restored and WAL logging is resumed. + * * @param mgr Database shared manager. * @throws IgniteCheckedException If failed. */ - public void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + default void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; /** - * @param mgr Database shared manager. * + * @param mgr + * @throws IgniteCheckedException */ - void afterMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + default void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; /** * @param mgr Database shared manager. */ - void afterInitialise(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException; + default void afterInitialise(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; /** * @param mgr Database shared manager. */ - void beforeStop(IgniteCacheDatabaseSharedManager mgr); + default void beforeStop(IgniteCacheDatabaseSharedManager mgr) {}; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index bf6464d3d4a91..f1a741d556c28 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -70,8 +70,6 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; -import org.apache.ignite.events.DiscoveryEvent; -import org.apache.ignite.events.EventType; import org.apache.ignite.failure.FailureContext; import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.GridKernalContext; @@ -82,9 +80,8 @@ import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; -import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider; -import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageUtils; @@ -96,19 +93,21 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; +import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware; import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.ExchangeActions; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.StoredCacheData; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; @@ -125,17 +124,16 @@ import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; +import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.processors.port.GridPortRecord; import org.apache.ignite.internal.util.GridMultiCollectionWrapper; -import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.future.CountDownFuture; import org.apache.ignite.internal.util.future.GridCompoundFuture; @@ -167,11 +165,9 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME; import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD; -import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID; import static org.apache.ignite.internal.util.IgniteUtils.checkpointBufferSize; /** @@ -182,6 +178,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** */ public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC"; + /** */ + public static final String IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP = "IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP"; + /** MemoryPolicyConfiguration name reserved for meta store. */ public static final String METASTORE_DATA_REGION_NAME = "metastoreMemPlc"; @@ -199,6 +198,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan private final String throttlingPolicyOverride = IgniteSystemProperties.getString( IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED); + /** */ + private final boolean skipCheckpointOnNodeStop = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, false); + /** Checkpoint lock hold count. */ private static final ThreadLocal CHECKPOINT_LOCK_HOLD_COUNT = new ThreadLocal() { @Override protected Integer initialValue() { @@ -454,6 +456,7 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi cfg.setInitialSize(storageCfg.getSystemRegionInitialSize()); cfg.setMaxSize(storageCfg.getSystemRegionMaxSize()); cfg.setPersistenceEnabled(true); + return cfg; } @@ -496,15 +499,7 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi .resolveFolders() .getLockedFileLockHolder(); - fileLockHolder = preLocked == null ? - new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log) : preLocked; - - if (log.isDebugEnabled()) - log.debug("Try to capture file lock [nodeId=" + - cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]"); - - if (!fileLockHolder.isLocked()) - fileLockHolder.tryLock(lockWaitTime); + acquireFileLock(preLocked); cleanupTempCheckpointDirectory(); @@ -532,17 +527,20 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi /** {@inheritDoc} */ @Override public void cleanupRestoredCaches() { - if (dataRegionMap == null) + if (dataRegionMap.isEmpty()) return; for (CacheGroupDescriptor grpDesc : cctx.cache().cacheGroupDescriptors().values()) { String regionName = grpDesc.config().getDataRegionName(); - DataRegion region = dataRegionMap.get(regionName == null ? DFLT_DATA_REG_DEFAULT_NAME : regionName); + DataRegion region = regionName != null ? dataRegionMap.get(regionName) : dfltDataRegion; if (region == null) continue; + if (log.isInfoEnabled()) + log.info("Page memory " + region + " for " + grpDesc + " has invalidated."); + int partitions = grpDesc.config().getAffinity().partitions(); if (region.pageMemory() instanceof PageMemoryEx) { @@ -550,6 +548,8 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi for (int partId = 0; partId < partitions; partId++) memEx.invalidate(grpDesc.groupId(), partId); + + memEx.invalidate(grpDesc.groupId(), PageIdAllocator.INDEX_PARTITION); } } @@ -578,6 +578,39 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi } } + /** + * @param preLocked Pre-locked file lock holder. + */ + private void acquireFileLock(FileLockHolder preLocked) throws IgniteCheckedException { + if (cctx.kernalContext().clientNode()) + return; + + fileLockHolder = preLocked == null ? + new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log) : preLocked; + + if (!fileLockHolder.isLocked()) { + if (log.isDebugEnabled()) + log.debug("Try to capture file lock [nodeId=" + + cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]"); + + fileLockHolder.tryLock(lockWaitTime); + } + } + + /** + * + */ + private void releaseFileLock() { + if (cctx.kernalContext().clientNode() || fileLockHolder == null) + return; + + if (log.isDebugEnabled()) + log.debug("Release file lock [nodeId=" + + cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]"); + + fileLockHolder.close(); + } + /** * Retreives checkpoint history form specified {@code dir}. * @@ -661,40 +694,18 @@ private void removeCheckpointFiles(CheckpointEntry cpEntry) throws IgniteChecked /** */ private void readMetastore() throws IgniteCheckedException { try { - DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration(); - - DataRegionConfiguration plcCfg = createMetastoreDataRegionConfig(memCfg); - - File allocPath = buildAllocPath(plcCfg); - - DirectMemoryProvider memProvider = allocPath == null ? - new UnsafeMemoryProvider(log) : - new MappedFileMemoryProvider( - log, - allocPath); - - DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(plcCfg); - - PageMemoryEx storePageMem = (PageMemoryEx)createPageMemory(memProvider, memCfg, plcCfg, memMetrics, false); - - DataRegion regCfg = new DataRegion(storePageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, storePageMem)); - CheckpointStatus status = readCheckpointStatus(); - cctx.pageStore().initializeForMetastorage(); - - storePageMem.start(); - checkpointReadLock(); try { - restoreMemory(status, true, storePageMem, Collections.emptySet()); + dataRegion(METASTORE_DATA_REGION_NAME).pageMemory().start(); - metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true); + performBinaryMemoryRestore(status, g -> MetaStorage.METASTORAGE_CACHE_ID == g, false); - metaStorage.init(this); + metaStorage = createMetastorage(true); - applyLastUpdates(status, true); + applyLogicalUpdates(status, g -> MetaStorage.METASTORAGE_CACHE_ID == g, false); fillWalDisabledGroups(); @@ -703,7 +714,7 @@ private void readMetastore() throws IgniteCheckedException { finally { metaStorage = null; - storePageMem.stop(true); + dataRegion(METASTORE_DATA_REGION_NAME).pageMemory().stop(false); cctx.pageStore().cleanupPageStoreIfMatch(new Predicate() { @Override public boolean test(Integer grpId) { @@ -729,21 +740,16 @@ private void readMetastore() throws IgniteCheckedException { snapshotMgr = cctx.snapshot(); - if (!cctx.kernalContext().clientNode() && persistenceCfg.getCheckpointThreads() > 1) { - asyncRunner = new IgniteThreadPoolExecutor( - CHECKPOINT_RUNNER_THREAD_PREFIX, - cctx.igniteInstanceName(), - persistenceCfg.getCheckpointThreads(), - persistenceCfg.getCheckpointThreads(), - 30_000, - new LinkedBlockingQueue<>() - ); - } - - if (checkpointer == null) + if (!cctx.kernalContext().clientNode() && checkpointer == null) checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log); super.onActivate(ctx); + + if (!cctx.kernalContext().clientNode()) { + initializeCheckpointPool(); + + finishRecovery(); + } } /** {@inheritDoc} */ @@ -760,6 +766,21 @@ private void readMetastore() throws IgniteCheckedException { stopping = false; } + /** + * + */ + private void initializeCheckpointPool() { + if (persistenceCfg.getCheckpointThreads() > 1) + asyncRunner = new IgniteThreadPoolExecutor( + CHECKPOINT_RUNNER_THREAD_PREFIX, + cctx.igniteInstanceName(), + persistenceCfg.getCheckpointThreads(), + persistenceCfg.getCheckpointThreads(), + 30_000, + new LinkedBlockingQueue() + ); + } + /** {@inheritDoc} */ @Override protected void registerMetricsMBeans(IgniteConfiguration cfg) { super.registerMetricsMBeans(cfg); @@ -798,8 +819,13 @@ private void readMetastore() throws IgniteCheckedException { }; } - /** {@inheritDoc} */ - @Override public void onDoneRestoreBinaryMemory() throws IgniteCheckedException { + /** + * Restores last valid WAL pointer and resumes logging from that pointer. + * Re-creates metastorage if needed. + * + * @throws IgniteCheckedException If failed. + */ + private void finishRecovery() throws IgniteCheckedException { assert !cctx.kernalContext().clientNode(); long time = System.currentTimeMillis(); @@ -810,37 +836,24 @@ private void readMetastore() throws IgniteCheckedException { for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) lsnr.beforeResumeWalLogging(this); - cctx.pageStore().initializeForMetastorage(); - - CheckpointStatus status = readCheckpointStatus(); - - // Binary memory should be recovered at startup. - assert !status.needRestoreMemory() : status; - - WALPointer statusEndPtr = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr; + // Try to resume logging since last finished checkpoint if possible. + if (walTail == null) { + CheckpointStatus status = readCheckpointStatus(); - // If binary memory recovery occurs resume from the last walTail in the other case from END checkpoint. - WALPointer walPtr = walTail == null ? statusEndPtr : walTail; + walTail = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr; + } - cctx.wal().resumeLogging(walPtr); + cctx.wal().resumeLogging(walTail); walTail = null; - metaStorage = new MetaStorage( - cctx, - dataRegionMap.get(METASTORE_DATA_REGION_NAME), - (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME), - false - ); - - // Init metastore only after WAL logging resumed. Can't do it earlier because - // MetaStorage first initialization also touches WAL, look at #isWalDeltaRecordNeeded. - metaStorage.init(this); + // Recreate metastorage to refresh page memory state after deactivation. + if (metaStorage == null) + metaStorage = createMetastorage(false); notifyMetastorageReadyForReadWrite(); - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) - lsnr.afterMemoryRestore(this); + U.log(log, "Finish recovery performed in " + (System.currentTimeMillis() - time) + " ms."); } catch (IgniteCheckedException e) { if (X.hasCause(e, StorageException.class, IOException.class)) @@ -850,17 +863,35 @@ private void readMetastore() throws IgniteCheckedException { } finally { checkpointReadUnlock(); - - U.log(log, "Resume logging performed in " + (System.currentTimeMillis() - time) + " ms."); } } /** - * @param cacheGrps Cache groups to restore. + * @param readOnly Metastorage read-only mode. + * @return Instance of Metastorage. + * @throws IgniteCheckedException If failed to create metastorage. + */ + private MetaStorage createMetastorage(boolean readOnly) throws IgniteCheckedException { + cctx.pageStore().initializeForMetastorage(); + + MetaStorage storage = new MetaStorage( + cctx, + dataRegion(METASTORE_DATA_REGION_NAME), + (DataRegionMetricsImpl) memMetricsMap.get(METASTORE_DATA_REGION_NAME), + readOnly + ); + + storage.init(this); + + return storage; + } + + /** + * @param cacheGroupsPredicate Cache groups to restore. * @return Last seen WAL pointer during binary memory recovery. * @throws IgniteCheckedException If failed. */ - protected WALPointer restoreBinaryMemory(Set cacheGrps) throws IgniteCheckedException { + private WALPointer restoreBinaryMemory(Predicate cacheGroupsPredicate) throws IgniteCheckedException { assert !cctx.kernalContext().clientNode(); long time = System.currentTimeMillis(); @@ -868,28 +899,23 @@ protected WALPointer restoreBinaryMemory(Set cacheGrps) throws IgniteCh checkpointReadLock(); try { - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) - lsnr.beforeBinaryMemoryRestore(this); - - cctx.pageStore().initializeForMetastorage(); - CheckpointStatus status = readCheckpointStatus(); // First, bring memory to the last consistent checkpoint state if needed. // This method should return a pointer to the last valid record in the WAL. - WALPointer tailWalPtr = restoreMemory(status, - false, - (PageMemoryEx)dataRegionMap.get(METASTORE_DATA_REGION_NAME).pageMemory(), - cacheGrps); + WALPointer restored = performBinaryMemoryRestore(status, cacheGroupsPredicate, true); - if (tailWalPtr == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { + if (restored == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { throw new StorageException("The memory cannot be restored. The critical part of WAL archive is missing " + - "[tailWalPtr=" + tailWalPtr + ", endPtr=" + status.endPtr + ']'); + "[tailWalPtr=" + restored + ", endPtr=" + status.endPtr + ']'); } - nodeStart(tailWalPtr); + nodeStart(restored); + + if (log.isInfoEnabled()) + log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms."); - return tailWalPtr; + return restored; } catch (IgniteCheckedException e) { if (X.hasCause(e, StorageException.class, IOException.class)) @@ -899,9 +925,6 @@ protected WALPointer restoreBinaryMemory(Set cacheGrps) throws IgniteCh } finally { checkpointReadUnlock(); - - if (log.isInfoEnabled()) - log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms."); } } @@ -1015,21 +1038,15 @@ public List> nodeStartedPointers() throws IgniteCheckedExce MBEAN_GROUP, MBEAN_NAME ); + + metaStorage = null; } /** {@inheritDoc} */ @Override protected void stop0(boolean cancel) { super.stop0(cancel); - if (!cctx.kernalContext().clientNode()) { - if (fileLockHolder != null) { - if (log.isDebugEnabled()) - log.debug("Release file lock [nodeId=" + - cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]"); - - fileLockHolder.close(); - } - } + releaseFileLock(); } /** */ @@ -1322,34 +1339,30 @@ private void shutdownCheckpointer(boolean cancel) { } /** {@inheritDoc} */ - @Override public boolean beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException { - DiscoveryEvent discoEvt = fut.firstEvent(); + @Override public void beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException { + // Try to restore partition states. + if (fut.localJoinExchange() || fut.activateCluster() + || (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart()))) { + U.doInParallel( + cctx.kernalContext().getSystemExecutorService(), + cctx.cache().cacheGroups(), + cacheGroup -> { + if (cacheGroup.isLocal()) + return; + + cctx.database().checkpointReadLock(); - boolean joinEvt = discoEvt.type() == EventType.EVT_NODE_JOINED; - - boolean locNode = discoEvt.eventNode().isLocal(); - - boolean isSrvNode = !cctx.kernalContext().clientNode(); - - boolean clusterInTransitionStateToActive = fut.activateCluster(); - - boolean restored = false; - - long time = System.currentTimeMillis(); - - // In case of cluster activation or local join restore, restore whole manager state. - if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) { - restoreState(); - - restored = true; - } - // In case of starting groups, restore partition states only for these groups. - else if (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart())) { - Set restoreGroups = fut.exchangeActions().cacheGroupsToStart().stream() - .map(actionData -> actionData.descriptor().groupId()) - .collect(Collectors.toSet()); + try { + cacheGroup.restorePartitionStates(Collections.emptyMap()); - restorePartitionStates(Collections.emptyMap(), restoreGroups); + if (cacheGroup.localStartVersion().equals(fut.initialVersion())) + cacheGroup.topology().afterStateRestored(fut.initialVersion()); + } + finally { + cctx.database().checkpointReadUnlock(); + } + } + ); } if (cctx.kernalContext().query().moduleEnabled()) { @@ -1366,11 +1379,6 @@ else if (acts.localJoinContext() != null && !F.isEmpty(acts.localJoinContext().c } } } - - if (log.isInfoEnabled()) - log.info("Logical recovery performed in " + (System.currentTimeMillis() - time) + " ms."); - - return restored; } /** @@ -1653,52 +1661,6 @@ private boolean safeToUpdatePageMemories() { CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() - 1); } - /** - * Restores from last checkpoint and applies WAL changes since this checkpoint. - * - * @throws IgniteCheckedException If failed to restore database status from WAL. - */ - private void restoreState() throws IgniteCheckedException { - try { - CheckpointStatus status = readCheckpointStatus(); - - checkpointReadLock(); - - try { - applyLastUpdates(status, false); - } - finally { - checkpointReadUnlock(); - } - - snapshotMgr.restoreState(); - } - catch (StorageException e) { - throw new IgniteCheckedException(e); - } - } - - /** - * Called when all partitions have been fully restored and pre-created on node start. - * - * Starts checkpointing process and initiates first checkpoint. - * - * @throws IgniteCheckedException If first checkpoint has failed. - */ - @Override public void onStateRestored() throws IgniteCheckedException { - long time = System.currentTimeMillis(); - - new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start(); - - CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started"); - - if (chp != null) - chp.cpBeginFut.get(); - - if (log.isInfoEnabled()) - log.info("Checkpointer initilialzation performed in " + (System.currentTimeMillis() - time) + " ms."); - } - /** {@inheritDoc} */ @Override public synchronized Map> reserveHistoryForExchange() { assert reservedForExchange == null : reservedForExchange; @@ -1997,39 +1959,146 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC if (kctx.clientNode()) return; - // Preform early regions startup before restoring state. - initAndStartRegions(kctx.config().getDataStorageConfiguration()); + checkpointReadLock(); + + try { + // Preform early regions startup before restoring state. + initAndStartRegions(kctx.config().getDataStorageConfiguration()); + + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) + lsnr.beforeBinaryMemoryRestore(this); + + log.info("Starting binary memory restore for: " + cctx.cache().cacheGroupDescriptors().keySet()); + + cctx.pageStore().initializeForMetastorage(); + + // Restore binary memory for all not WAL disabled cache groups. + WALPointer restored = restoreBinaryMemory( + g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g) + ); + + if (restored != null) + U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restored + ']'); + + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) + lsnr.afterBinaryMemoryRestore(this); + + cctx.wal().resumeLogging(restored); + + // We should log this record to ensure that node start marker pointer will be found in compacted segment. + cctx.wal().log(new MemoryRecoveryRecord(System.currentTimeMillis())); - // Only presistence caches to start. - for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) { - if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration())) - storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration())); + assert metaStorage == null; + + metaStorage = createMetastorage(false); + + CheckpointStatus status = readCheckpointStatus(); + + RestoreLogicalState logicalState = applyLogicalUpdates( + status, + g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g), + true + ); + + // Restore state for all groups. + restorePartitionStates(cctx.cache().cacheGroups(), logicalState.partitionRecoveryStates); + + walTail = tailPointer(logicalState.lastRead); + + cctx.wal().onDeActivate(kctx); } + catch (IgniteCheckedException e) { + releaseFileLock(); - final WALPointer restoredPtr = restoreBinaryMemory(cctx.cache().cacheGroupDescriptors().keySet()); + throw e; + } + finally { + checkpointReadUnlock(); + } + } - walTail = restoredPtr; + /** + * Calculates tail pointer for WAL at the end of logical recovery. + * + * @param from Start replay WAL from. + * @return Tail pointer. + * @throws IgniteCheckedException If failed. + */ + private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { + WALPointer lastRead = from; + + try (WALIterator it = cctx.wal().replay(from)) { + while (it.hasNextX()) { + IgniteBiTuple rec = it.nextX(); - if (restoredPtr != null) - U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restoredPtr + ']'); + if (rec == null) + break; + + lastRead = rec.get1(); + } + } + + return lastRead != null ? lastRead.next() : null; + } + + /** + * @param forGroups Cache groups. + * @param partitionStates Partition states. + * @throws IgniteCheckedException If failed. + */ + private void restorePartitionStates( + Collection forGroups, + Map partitionStates + ) throws IgniteCheckedException { + long startRestorePart = U.currentTimeMillis(); + + if (log.isInfoEnabled()) + log.info("Restoring partition state for local groups."); + + long totalProcessed = 0; + + for (CacheGroupContext grp : forGroups) + totalProcessed += grp.restorePartitionStates(partitionStates); + + if (log.isInfoEnabled()) + log.info("Finished restoring partition state for local groups [" + + "groupsProcessed" + forGroups.size() + + "partitionsProcessed=" + totalProcessed + + ", time=" + (U.currentTimeMillis() - startRestorePart) + "ms]"); + } + + /** + * Called when all partitions have been fully restored and pre-created on node start. + * + * Starts checkpointing process and initiates first checkpoint. + * + * @throws IgniteCheckedException If first checkpoint has failed. + */ + @Override public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException { + long time = System.currentTimeMillis(); + + new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start(); + + CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started"); + + if (chp != null) + chp.cpBeginFut.get(); + + if (log.isInfoEnabled()) + log.info("Checkpointer initilialzation performed in " + (System.currentTimeMillis() - time) + " ms."); } /** * @param status Checkpoint status. - * @param metastoreOnly If {@code True} restores Metastorage only. - * @param storePageMem Metastore page memory. - * @param cacheGrps Cache groups to restore. + * @param cacheGroupsPredicate Cache groups to restore. * @throws IgniteCheckedException If failed. * @throws StorageException In case I/O error occurred during operations with storage. */ - @Nullable private WALPointer restoreMemory( + private WALPointer performBinaryMemoryRestore( CheckpointStatus status, - boolean metastoreOnly, - PageMemoryEx storePageMem, - Set cacheGrps + Predicate cacheGroupsPredicate, + boolean finalizeState ) throws IgniteCheckedException { - assert !metastoreOnly || storePageMem != null; - if (log.isInfoEnabled()) log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']'); @@ -2037,8 +2106,9 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC boolean apply = status.needRestoreMemory(); if (apply) { - U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " + - "finish checkpoint on node start."); + if (finalizeState) + U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " + + "finish checkpoint on node start."); cctx.pageStore().beginRecover(); } @@ -2049,16 +2119,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC long lastArchivedSegment = cctx.wal().lastArchivedSegment(); - RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, log); - - // Always perform recovery at least meta storage cache. - Set restoreGrps = new HashSet<>(Collections.singletonList(METASTORAGE_CACHE_ID)); - - if (!metastoreOnly && !F.isEmpty(cacheGrps)) { - restoreGrps.addAll(cacheGrps.stream() - .filter(g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g)) - .collect(Collectors.toSet())); - } + RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, cacheGroupsPredicate); int applied = 0; @@ -2078,29 +2139,30 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC // several repetitive restarts and the same pages may have changed several times. int grpId = pageRec.fullPageId().groupId(); - if (restoreGrps.contains(grpId)) { - long pageId = pageRec.fullPageId().pageId(); + long pageId = pageRec.fullPageId().pageId(); - PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId); + PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId); - long page = pageMem.acquirePage(grpId, pageId, true); + if (pageMem == null) + break; - try { - long pageAddr = pageMem.writeLock(grpId, pageId, page); + long page = pageMem.acquirePage(grpId, pageId, true); - try { - PageUtils.putBytes(pageAddr, 0, pageRec.pageData()); - } - finally { - pageMem.writeUnlock(grpId, pageId, page, null, true, true); - } + try { + long pageAddr = pageMem.writeLock(grpId, pageId, page); + + try { + PageUtils.putBytes(pageAddr, 0, pageRec.pageData()); } finally { - pageMem.releasePage(grpId, pageId, page); + pageMem.writeUnlock(grpId, pageId, page, null, true, true); } - - applied++; } + finally { + pageMem.releasePage(grpId, pageId, page); + } + + applied++; } break; @@ -2111,9 +2173,6 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC { int grpId = metaStateRecord.groupId(); - if (!restoreGrps.contains(grpId)) - continue; - int partId = metaStateRecord.partitionId(); GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(metaStateRecord.state()); @@ -2132,10 +2191,10 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC { int grpId = destroyRecord.groupId(); - if (!restoreGrps.contains(grpId)) - continue; + PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId); - PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId); + if (pageMem == null) + break; pageMem.invalidate(grpId, destroyRecord.partitionId()); @@ -2150,37 +2209,38 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC int grpId = r.groupId(); - if (restoreGrps.contains(grpId)) { - long pageId = r.pageId(); + long pageId = r.pageId(); - PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId); + PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId); - // Here we do not require tag check because we may be applying memory changes after - // several repetitive restarts and the same pages may have changed several times. - long page = pageMem.acquirePage(grpId, pageId, true); + if (pageMem == null) + break; - try { - long pageAddr = pageMem.writeLock(grpId, pageId, page); + // Here we do not require tag check because we may be applying memory changes after + // several repetitive restarts and the same pages may have changed several times. + long page = pageMem.acquirePage(grpId, pageId, true); - try { - r.applyDelta(pageMem, pageAddr); - } - finally { - pageMem.writeUnlock(grpId, pageId, page, null, true, true); - } + try { + long pageAddr = pageMem.writeLock(grpId, pageId, page); + + try { + r.applyDelta(pageMem, pageAddr); } finally { - pageMem.releasePage(grpId, pageId, page); + pageMem.writeUnlock(grpId, pageId, page, null, true, true); } - - applied++; } + finally { + pageMem.releasePage(grpId, pageId, page); + } + + applied++; } } } } - if (metastoreOnly) + if (!finalizeState) return null; WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); @@ -2200,7 +2260,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC cpHistory.initialize(retreiveHistory()); - return lastReadPtr == null ? null : lastReadPtr.next(); + return lastReadPtr != null ? lastReadPtr.next() : null; } /** @@ -2211,6 +2271,9 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC * @throws IgniteCheckedException if no DataRegion is configured for a name obtained from cache descriptor. */ private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedException { + if (grpId == MetaStorage.METASTORAGE_CACHE_ID) + return (PageMemoryEx)dataRegion(METASTORE_DATA_REGION_NAME).pageMemory(); + // TODO IGNITE-7792 add generic mapping. if (grpId == TxLog.TX_LOG_CACHE_ID) return (PageMemoryEx)dataRegion(TxLog.TX_LOG_CACHE_NAME).pageMemory(); @@ -2221,7 +2284,7 @@ private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedE CacheGroupDescriptor desc = sharedCtx.cache().cacheGroupDescriptors().get(grpId); if (desc == null) - throw new IgniteCheckedException("Failed to find cache group descriptor [grpId=" + grpId + ']'); + return null; String memPlcName = desc.config().getDataRegionName(); @@ -2234,13 +2297,13 @@ private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedE * @param it WalIterator. * @param recPredicate Wal record filter. * @param entryPredicate Entry filter. - * @param partStates Partition to restore state. + * @param partitionRecoveryStates Partition to restore state. */ public void applyUpdatesOnRecovery( @Nullable WALIterator it, IgnitePredicate> recPredicate, IgnitePredicate entryPredicate, - Map, T2> partStates + Map partitionRecoveryStates ) throws IgniteCheckedException { cctx.walState().runWithOutWAL(() -> { if (it != null) { @@ -2298,7 +2361,7 @@ else if (log != null) checkpointReadLock(); try { - restorePartitionStates(partStates, null); + restorePartitionStates(cctx.cache().cacheGroups(), partitionRecoveryStates); } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -2311,31 +2374,30 @@ else if (log != null) /** * @param status Last registered checkpoint status. - * @param metastoreOnly If {@code True} only records related to metastorage will be processed. * @throws IgniteCheckedException If failed to apply updates. * @throws StorageException If IO exception occurred while reading write-ahead log. */ - private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) throws IgniteCheckedException { + private RestoreLogicalState applyLogicalUpdates( + CheckpointStatus status, + Predicate cacheGroupsPredicate, + boolean skipFieldLookup + ) throws IgniteCheckedException { if (log.isInfoEnabled()) log.info("Applying lost cache updates since last checkpoint record [lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']'); - if (!metastoreOnly) + if (skipFieldLookup) cctx.kernalContext().query().skipFieldLookup(true); long lastArchivedSegment = cctx.wal().lastArchivedSegment(); - RestoreLogicalState restoreLogicalState = new RestoreLogicalState(lastArchivedSegment, log); + RestoreLogicalState restoreLogicalState = new RestoreLogicalState(lastArchivedSegment, cacheGroupsPredicate); long start = U.currentTimeMillis(); - int applied = 0; - Collection ignoreGrps = metastoreOnly ? Collections.emptySet() : - F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps); + int applied = 0; try (WALIterator it = cctx.wal().replay(status.startPtr)) { - Map, T2> partStates = new HashMap<>(); - while (it.hasNextX()) { WALRecord rec = restoreLogicalState.next(it); @@ -2344,9 +2406,6 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th switch (rec.type()) { case DATA_RECORD: - if (metastoreOnly) - continue; - DataRecord dataRec = (DataRecord)rec; for (DataEntry dataEntry : dataRec.writeEntries()) { @@ -2358,27 +2417,27 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th if (cacheDesc == null) continue; - if (!ignoreGrps.contains(cacheDesc.groupId())) { - GridCacheContext cacheCtx = cctx.cacheContext(cacheId); + GridCacheContext cacheCtx = cctx.cacheContext(cacheId); - applyUpdate(cacheCtx, dataEntry); + applyUpdate(cacheCtx, dataEntry); - applied++; - } + applied++; } break; case PART_META_UPDATE_STATE: - if (metastoreOnly) - continue; - PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec; - if (!ignoreGrps.contains(metaStateRecord.groupId())) { - partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()), - new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter())); - } + GroupPartitionId groupPartitionId = new GroupPartitionId( + metaStateRecord.groupId(), metaStateRecord.partitionId() + ); + + PartitionRecoverState state = new PartitionRecoverState( + (int)metaStateRecord.state(), metaStateRecord.updateCounter() + ); + + restoreLogicalState.partitionRecoveryStates.put(groupPartitionId, state); break; @@ -2392,13 +2451,13 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: - if (metastoreOnly) - continue; - PageDeltaRecord rec0 = (PageDeltaRecord) rec; PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId()); + if (pageMem == null) + break; + long page = pageMem.acquirePage(rec0.groupId(), rec0.pageId(), true); try { @@ -2421,166 +2480,17 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th // Skip other records. } } - - if (!metastoreOnly) { - long startRestorePart = U.currentTimeMillis(); - - if (log.isInfoEnabled()) - log.info("Restoring partition state for local groups [cntPartStateWal=" - + partStates.size() + ", lastCheckpointId=" + status.cpStartId + ']'); - - long proc = restorePartitionStates(partStates, null); - - if (log.isInfoEnabled()) - log.info("Finished restoring partition state for local groups [cntProcessed=" + proc + - ", cntPartStateWal=" + partStates.size() + - ", time=" + (U.currentTimeMillis() - startRestorePart) + "ms]"); - } } finally { - if (!metastoreOnly) + if (skipFieldLookup) cctx.kernalContext().query().skipFieldLookup(false); } if (log.isInfoEnabled()) log.info("Finished applying WAL changes [updatesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + " ms]"); - } - /** - * Initializes not empty partitions and restores their state from page memory or WAL. - * Partition states presented in page memory may be overriden by states restored from WAL {@code partStates}. - * - * @param partStates Partition states restored from WAL. - * @param onlyForGroups If not {@code null} restore states only for specified cache groups. - * @return cntParts Count of partitions processed. - * @throws IgniteCheckedException If failed to restore partition states. - */ - private long restorePartitionStates( - Map, T2> partStates, - @Nullable Set onlyForGroups - ) throws IgniteCheckedException { - long cntParts = 0; - - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { - if (grp.isLocal() || !grp.affinityNode()) { - // Local cache has no partitions and its states. - continue; - } - - if (!grp.dataRegion().config().isPersistenceEnabled()) - continue; - - if (onlyForGroups != null && !onlyForGroups.contains(grp.groupId())) - continue; - - int grpId = grp.groupId(); - - PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory(); - - for (int i = 0; i < grp.affinity().partitions(); i++) { - T2 restore = partStates.get(new T2<>(grpId, i)); - - if (storeMgr.exists(grpId, i)) { - storeMgr.ensure(grpId, i); - - if (storeMgr.pages(grpId, i) <= 1) - continue; - - if (log.isDebugEnabled()) - log.debug("Creating partition on recovery (exists in page store) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + i + "]"); - - GridDhtLocalPartition part = grp.topology().forceCreatePartition(i); - - assert part != null; - - // TODO: https://issues.apache.org/jira/browse/IGNITE-6097 - grp.offheap().onPartitionInitialCounterUpdated(i, 0); - - checkpointReadLock(); - - try { - long partMetaId = pageMem.partitionMetaPageId(grpId, i); - long partMetaPage = pageMem.acquirePage(grpId, partMetaId); - - try { - long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage); - - boolean changed = false; - - try { - PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr); - - if (restore != null) { - int stateId = restore.get1(); - - io.setPartitionState(pageAddr, (byte)stateId); - - changed = updateState(part, stateId); - - if (stateId == GridDhtPartitionState.OWNING.ordinal() - || (stateId == GridDhtPartitionState.MOVING.ordinal() - && part.initialUpdateCounter() < restore.get2())) { - part.initialUpdateCounter(restore.get2()); - - changed = true; - } - - if (log.isDebugEnabled()) - log.debug("Restored partition state (from WAL) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() + - "updCntr=" + part.initialUpdateCounter() + "]"); - } - else { - changed = updateState(part, (int) io.getPartitionState(pageAddr)); - - if (log.isDebugEnabled()) - log.debug("Restored partition state (from page memory) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() + - "updCntr=" + part.initialUpdateCounter() + "]"); - } - } - finally { - pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed); - } - } - finally { - pageMem.releasePage(grpId, partMetaId, partMetaPage); - } - } - finally { - checkpointReadUnlock(); - } - } - else if (restore != null) { - if (log.isDebugEnabled()) - log.debug("Creating partition on recovery (exists in WAL) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + i + "]"); - - GridDhtLocalPartition part = grp.topology().forceCreatePartition(i); - - assert part != null; - - // TODO: https://issues.apache.org/jira/browse/IGNITE-6097 - grp.offheap().onPartitionInitialCounterUpdated(i, 0); - - updateState(part, restore.get1()); - - if (log.isDebugEnabled()) - log.debug("Restored partition state (from WAL) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() + - "updCntr=" + part.initialUpdateCounter() + "]"); - } - - cntParts++; - } - - // After partition states are restored, it is necessary to update internal data structures in topology. - grp.topology().afterStateRestored(grp.topology().lastTopologyChangeVersion()); - } - - return cntParts; + return restoreLogicalState; } /** @@ -2595,25 +2505,6 @@ public void onWalTruncated(WALPointer highBound) throws IgniteCheckedException { removeCheckpointFiles(cp); } - /** - * @param part Partition to restore state for. - * @param stateId State enum ordinal. - * @return Updated flag. - */ - private boolean updateState(GridDhtLocalPartition part, int stateId) { - if (stateId != -1) { - GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId); - - assert state != null; - - part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state); - - return true; - } - - return false; - } - /** * @param cacheCtx Cache context to apply an update. * @param dataEntry Data entry to apply. @@ -3135,6 +3026,13 @@ protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log) while (!isCancelled()) { waitCheckpointEvent(); + if (skipCheckpointOnNodeStop && (isStopping() || shutdownNow)) { + if (log.isInfoEnabled()) + log.warning("Skipping last checkpoint because node is stopping."); + + return; + } + GridFutureAdapter enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied; if (enableChangeApplied != null) { @@ -3566,7 +3464,7 @@ private void cancelOrWaitPartitionDestroy(int grpId, int partId) throws IgniteCh if (req != null) req.waitCompleted(); - if (log.isDebugEnabled()) + if (req != null && log.isDebugEnabled()) log.debug("Partition file destroy has cancelled [grpId=" + grpId + ", partId=" + partId + "]"); } @@ -4129,7 +4027,7 @@ else if (grpId == TxLog.TX_LOG_CACHE_ID) else { CacheGroupContext grp = context().cache().cacheGroup(grpId); - DataRegion region = grp != null ?grp .dataRegion() : null; + DataRegion region = grp != null ? grp.dataRegion() : null; if (region == null || !region.config().isPersistenceEnabled()) continue; @@ -4637,10 +4535,10 @@ public boolean isCheckpointInapplicableForWalRebalance(Long cpTs, int grpId) thr * */ private void fillWalDisabledGroups() { - MetaStorage meta = cctx.database().metaStorage(); + assert metaStorage != null; try { - Set keys = meta.readForPredicate(WAL_KEY_PREFIX_PRED).keySet(); + Set keys = metaStorage.readForPredicate(WAL_KEY_PREFIX_PRED).keySet(); if (keys.isEmpty()) return; @@ -4705,23 +4603,26 @@ else if (key.startsWith(WAL_GLOBAL_KEY_PREFIX)) /** * Abstract class for create restore context. */ - public abstract static class RestoreStateContext { - /** */ - protected final IgniteLogger log; - + private abstract class RestoreStateContext { /** Last archived segment. */ protected final long lastArchivedSegment; /** Last read record WAL pointer. */ protected FileWALPointer lastRead; + /** Only {@link WalRecordCacheGroupAware} records satisfied this predicate will be applied. */ + private final Predicate cacheGroupPredicate; + + /** Set to {@code true} if data records should be skipped. */ + private final boolean skipDataRecords; + /** * @param lastArchivedSegment Last archived segment index. - * @param log Ignite logger. */ - public RestoreStateContext(long lastArchivedSegment, IgniteLogger log) { + public RestoreStateContext(long lastArchivedSegment, Predicate cacheGroupPredicate, boolean skipDataRecords) { this.lastArchivedSegment = lastArchivedSegment; - this.log = log; + this.cacheGroupPredicate = cacheGroupPredicate; + this.skipDataRecords = skipDataRecords; } /** @@ -4733,17 +4634,63 @@ public RestoreStateContext(long lastArchivedSegment, IgniteLogger log) { */ public WALRecord next(WALIterator it) throws IgniteCheckedException { try { - IgniteBiTuple tup = it.nextX(); + for (;;) { + if (!it.hasNextX()) + return null; - WALRecord rec = tup.get2(); + IgniteBiTuple tup = it.nextX(); - WALPointer ptr = tup.get1(); + if (tup == null) + return null; - lastRead = (FileWALPointer)ptr; + WALRecord rec = tup.get2(); - rec.position(ptr); + WALPointer ptr = tup.get1(); - return rec; + lastRead = (FileWALPointer)ptr; + + rec.position(ptr); + + // Filter out records. + if (rec instanceof WalRecordCacheGroupAware) { + WalRecordCacheGroupAware groupAwareRecord = (WalRecordCacheGroupAware) rec; + + if (!cacheGroupPredicate.test(groupAwareRecord.groupId())) + continue; + } + + switch (rec.type()) { + case METASTORE_DATA_RECORD: + case DATA_RECORD: + if (skipDataRecords) + continue; + + if (rec instanceof DataRecord) { + DataRecord dataRecord = (DataRecord) rec; + + // Filter data entries by group id. + List filteredEntries = dataRecord.writeEntries().stream() + .filter(entry -> { + if (entry == null) + return false; + + int cacheId = entry.cacheId(); + + return cctx != null && cctx.cacheContext(cacheId) != null && cacheGroupPredicate.test(cctx.cacheContext(cacheId).groupId()); + }) + .collect(Collectors.toList()); + + dataRecord.setWriteEntries(filteredEntries); + } + + break; + + default: + break; + } + + return rec; + } } catch (IgniteCheckedException e) { boolean throwsCRCError = throwsCRCError(); @@ -4783,7 +4730,7 @@ public boolean throwsCRCError(){ /** * Restore memory context. Tracks the safety of binary recovery. */ - public static class RestoreBinaryState extends RestoreStateContext { + private class RestoreBinaryState extends RestoreStateContext { /** Checkpoint status. */ private final CheckpointStatus status; @@ -4793,13 +4740,12 @@ public static class RestoreBinaryState extends RestoreStateContext { /** * @param status Checkpoint status. * @param lastArchivedSegment Last archived segment index. - * @param log Ignite logger. */ - public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, IgniteLogger log) { - super(lastArchivedSegment, log); + public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, Predicate cacheGroupsPredicate) { + super(lastArchivedSegment, cacheGroupsPredicate, true); this.status = status; - needApplyBinaryUpdates = status.needRestoreMemory(); + this.needApplyBinaryUpdates = status.needRestoreMemory(); } /** @@ -4859,13 +4805,15 @@ public boolean needApplyBinaryUpdate() { /** * Restore logical state context. Tracks the safety of logical recovery. */ - public static class RestoreLogicalState extends RestoreStateContext { + private class RestoreLogicalState extends RestoreStateContext { + /** States of partitions recovered during applying logical updates. */ + private final Map partitionRecoveryStates = new HashMap<>(); + /** * @param lastArchivedSegment Last archived segment index. - * @param log Ignite logger. */ - public RestoreLogicalState(long lastArchivedSegment, IgniteLogger log) { - super(lastArchivedSegment, log); + public RestoreLogicalState(long lastArchivedSegment, Predicate cacheGroupsPredicate) { + super(lastArchivedSegment, cacheGroupsPredicate, false); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 21bd45482cd14..4966bcaa0290d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import javax.management.InstanceNotFoundException; import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.DataStorageMetrics; @@ -45,6 +46,7 @@ import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; @@ -91,13 +93,19 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024; /** {@code True} to reuse memory on deactive. */ - private final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); + protected final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE); + + /** */ + protected final Map dataRegionMap = new ConcurrentHashMap<>(); + + /** Stores memory providers eligible for reuse. */ + private final Map memProviderMap = new ConcurrentHashMap<>(); /** */ private static final String MBEAN_GROUP_NAME = "DataRegionMetrics"; /** */ - protected volatile Map dataRegionMap; + protected final Map memMetricsMap = new ConcurrentHashMap<>(); /** */ private volatile boolean dataRegionsInitialized; @@ -105,9 +113,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** */ private volatile boolean dataRegionsStarted; - /** */ - protected Map memMetricsMap; - /** */ protected DataRegion dfltDataRegion; @@ -123,8 +128,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** First eviction was warned flag. */ private volatile boolean firstEvictWarn; - /** Stores memory providers eligible for reuse. */ - private Map memProviderMap; /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { @@ -266,6 +269,17 @@ public int pageSize() { return pageSize; } + /** + * + */ + private void startDataRegions() { + for (DataRegion region : dataRegionMap.values()) { + region.pageMemory().start(); + + region.evictionTracker().start(); + } + } + /** * @param memCfg Database config. * @throws IgniteCheckedException If failed to initialize swap path. @@ -288,12 +302,6 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe protected void initDataRegions0(DataStorageConfiguration memCfg) throws IgniteCheckedException { DataRegionConfiguration[] dataRegionCfgs = memCfg.getDataRegionConfigurations(); - int dataRegions = dataRegionCfgs == null ? 0 : dataRegionCfgs.length; - - dataRegionMap = U.newHashMap(3 + dataRegions); - memMetricsMap = U.newHashMap(3 + dataRegions); - memProviderMap = reuseMemory ? U.newHashMap(3 + dataRegions) : null; - if (dataRegionCfgs != null) { for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs) addDataRegion(memCfg, dataRegionCfg, dataRegionCfg.isPersistenceEnabled()); @@ -346,14 +354,14 @@ public void addDataRegion( DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(dataRegionCfg, freeSpaceProvider(dataRegionCfg)); - DataRegion memPlc = initMemory(dataStorageCfg, dataRegionCfg, memMetrics, trackable); + DataRegion region = initMemory(dataStorageCfg, dataRegionCfg, memMetrics, trackable); - dataRegionMap.put(dataRegionName, memPlc); + dataRegionMap.put(dataRegionName, region); memMetricsMap.put(dataRegionName, memMetrics); if (dataRegionName.equals(dfltMemPlcName)) - dfltDataRegion = memPlc; + dfltDataRegion = region; else if (dataRegionName.equals(DFLT_DATA_REG_DEFAULT_NAME)) U.warn(log, "Data Region with name 'default' isn't used as a default. " + "Please, check Data Region configuration."); @@ -688,13 +696,6 @@ public DataStorageMetrics persistentStoreMetrics() { return null; } - /** - * @throws IgniteCheckedException If fails. - */ - public void onDoneRestoreBinaryMemory() throws IgniteCheckedException { - // No-op. - } - /** * Creates file with current timestamp and specific "node-started.bin" suffix * and writes into memory recovery pointer. @@ -729,7 +730,7 @@ public DataRegion dataRegion(String memPlcName) throws IgniteCheckedException { if (memPlcName == null) return dfltDataRegion; - if (dataRegionMap == null) + if (dataRegionMap.isEmpty()) return null; DataRegion plc; @@ -856,11 +857,9 @@ public void waitForCheckpoint(String reason) throws IgniteCheckedException { /** * @param discoEvt Before exchange for the given discovery event. - * - * @return {@code True} if partitions have been restored from persistent storage. */ - public boolean beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException { - return false; + public void beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException { + } /** @@ -878,7 +877,7 @@ public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedExcep * * @throws IgniteCheckedException If failed. */ - public void onStateRestored() throws IgniteCheckedException { + public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException { // No-op. } @@ -1026,7 +1025,7 @@ private DirectMemoryProvider createOrReuseMemoryProvider(DataRegionConfiguration * * @return {@code True} if policy supports memory reuse. */ - private boolean supportsMemoryReuse(DataRegionConfiguration plcCfg) { + public boolean supportsMemoryReuse(DataRegionConfiguration plcCfg) { return reuseMemory && plcCfg.getSwapPath() == null; } @@ -1209,11 +1208,7 @@ private void startDataRegions(DataStorageConfiguration cfg) throws IgniteChecked registerMetricsMBeans(cctx.gridConfig()); - for (DataRegion memPlc : dataRegionMap.values()) { - memPlc.pageMemory().start(); - - memPlc.evictionTracker().start(); - } + startDataRegions(); initPageMemoryDataStructures(cfg); @@ -1234,33 +1229,26 @@ private void onDeActivate(boolean shutdown) { for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) lsnr.beforeStop(this); - if (dataRegionMap != null) { - for (DataRegion memPlc : dataRegionMap.values()) { - memPlc.pageMemory().stop(shutdown); + for (DataRegion region : dataRegionMap.values()) { + region.pageMemory().stop(shutdown); - memPlc.evictionTracker().stop(); + region.evictionTracker().stop(); - unregisterMetricsMBean( - cctx.gridConfig(), - MBEAN_GROUP_NAME, - memPlc.memoryMetrics().getName() - ); + unregisterMetricsMBean( + cctx.gridConfig(), + MBEAN_GROUP_NAME, + region.memoryMetrics().getName() + ); } - dataRegionMap.clear(); + dataRegionMap.clear(); - dataRegionMap = null; + if (shutdown && memProviderMap != null) + memProviderMap.clear(); - if (shutdown && memProviderMap != null) { - memProviderMap.clear(); + dataRegionsInitialized = false; - memProviderMap = null; - } - - dataRegionsInitialized = false; - - dataRegionsStarted = false; - } + dataRegionsStarted = false; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java index 104697e810e58..e0c545b8a87c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java @@ -22,10 +22,6 @@ import java.nio.channels.AsynchronousFileChannel; import java.nio.file.OpenOption; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.WRITE; - /** * File I/O factory which uses {@link AsynchronousFileChannel} based implementation of FileIO. */ @@ -36,11 +32,6 @@ public class AsyncFileIOFactory implements FileIOFactory { /** Thread local channel future holder. */ private transient volatile ThreadLocal holder = initHolder(); - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { if (holder == null) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java index 336aab6fa85dd..b4b0389ef2c39 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIOFactory.java @@ -75,13 +75,6 @@ public class EncryptedFileIOFactory implements FileIOFactory { this.encSpi = encSpi; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - FileIO io = plainIOFactory.create(file); - - return new EncryptedFileIO(io, groupId, pageSize, headerSize, encMgr, encSpi); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { FileIO io = plainIOFactory.create(file, modes); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java index 27351853269db..b236000d07c72 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java @@ -22,6 +22,10 @@ import java.io.Serializable; import java.nio.file.OpenOption; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.WRITE; + /** * {@link FileIO} factory definition. */ @@ -33,7 +37,9 @@ public interface FileIOFactory extends Serializable { * @return File I/O interface. * @throws IOException If I/O interface creation was failed. */ - public FileIO create(File file) throws IOException; + default FileIO create(File file) throws IOException{ + return create(file, CREATE, READ, WRITE); + }; /** * Creates I/O interface for file with specified mode. @@ -43,5 +49,5 @@ public interface FileIOFactory extends Serializable { * @return File I/O interface. * @throws IOException If I/O interface creation was failed. */ - public FileIO create(File file, OpenOption... modes) throws IOException; + FileIO create(File file, OpenOption... modes) throws IOException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index fdf4705b5b177..16d74c33c66e6 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -258,6 +258,10 @@ private long checkFile(FileIO fileIO) throws IOException { + ", delete=" + delete + "]", e); } finally { + allocatedTracker.updateTotalAllocatedPages(-1L * allocated.getAndSet(0) / pageSize); + + inited = false; + lock.writeLock().unlock(); } } @@ -542,7 +546,8 @@ private void reinit(FileIO fileIO) throws IOException { long off = pageOffset(pageId); assert (off >= 0 && off <= allocated.get()) || recover : - "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId); + "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + + ", pageId=" + U.hexLong(pageId) + ", file=" + cfgFile.getPath(); assert pageBuf.capacity() == pageSize; assert pageBuf.position() == 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java index 856ba1c749b45..3fa3e2dded52d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java @@ -21,10 +21,6 @@ import java.io.IOException; import java.nio.file.OpenOption; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.WRITE; - /** * File I/O factory which provides RandomAccessFileIO implementation of FileIO. */ @@ -32,11 +28,6 @@ public class RandomAccessFileIOFactory implements FileIOFactory { /** */ private static final long serialVersionUID = 0L; - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { return new RandomAccessFileIO(file, modes); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java index 8ab418c4f4d5e..12cc4468fc010 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java @@ -33,7 +33,7 @@ public interface MetastorageLifecycleListener { * * @param metastorage Read-only meta storage. */ - public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException; + default void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException { }; /** * Fully functional metastore capable of performing reading and writing operations. @@ -43,5 +43,5 @@ public interface MetastorageLifecycleListener { * * @param metastorage Fully functional meta storage. */ - public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException; + default void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException { }; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java new file mode 100644 index 0000000000000..4c7e4d7323011 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.partstate; + +/** + * Class holds state of partition during recovery process. + */ +public class PartitionRecoverState { + /** State id. */ + private final int stateId; + + /** Update counter. */ + private final long updateCounter; + + /** + * @param stateId State id. + * @param updateCounter Update counter. + */ + public PartitionRecoverState(int stateId, long updateCounter) { + this.stateId = stateId; + this.updateCounter = updateCounter; + } + + /** + * + */ + public int stateId() { + return stateId; + } + + /** + * + */ + public long updateCounter() { + return updateCounter; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java index 16cc8f560c278..75aa983abe62d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java @@ -94,10 +94,6 @@ public boolean partitionsAreFrozen(CacheGroupContext grp) { /** * */ - public void restoreState() throws IgniteCheckedException { - // No-op. - } - public boolean snapshotOperationInProgress(){ return false; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 31dbb968e9933..53ea14f7b0ad5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -81,9 +81,12 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord; +import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; +import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext; @@ -529,6 +532,23 @@ public void setFileIOFactory(FileIOFactory ioFactory) { } } + /** + * + */ + private void startArchiverAndCompressor() { + if (isArchiverEnabled()) { + assert archiver != null; + + new IgniteThread(archiver).start(); + } + + if (walSegmentSyncWorker != null) + new IgniteThread(walSegmentSyncWorker).start(); + + if (compressor != null) + compressor.start(); + } + /** * Archiver can be not created, all files will be written to WAL folder, using absolute segment index. * @@ -628,11 +648,17 @@ private void checkWalConfiguration() throws IgniteCheckedException { if (archiver != null) archiver.shutdown(); - if (compressor != null) + if (compressor != null) { compressor.shutdown(); - if (decompressor != null) + compressor = null; + } + + if (decompressor != null) { decompressor.shutdown(); + + decompressor = null; + } } catch (Exception e) { U.error(log, "Failed to gracefully close WAL segment: " + this.currHnd.fileIO, e); @@ -646,20 +672,6 @@ private void checkWalConfiguration() throws IgniteCheckedException { " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); start0(); - - if (!cctx.kernalContext().clientNode()) { - if (isArchiverEnabled()) { - assert archiver != null; - - new IgniteThread(archiver).start(); - } - - if (walSegmentSyncWorker != null) - new IgniteThread(walSegmentSyncWorker).start(); - - if (compressor != null) - compressor.start(); - } } /** {@inheritDoc} */ @@ -687,6 +699,9 @@ private void checkWalConfiguration() throws IgniteCheckedException { @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException { assert currHnd == null; assert lastPtr == null || lastPtr instanceof FileWALPointer; + + startArchiverAndCompressor(); + assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) : "Trying to restore FileWriteHandle on deactivated write ahead log manager"; @@ -810,6 +825,11 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { if (serializer == null || mode == WALMode.NONE) return null; + // Only delta-records, page snapshots and memory recovery are allowed to write in recovery mode. + if (cctx.kernalContext().recoveryMode() && + !(rec instanceof PageDeltaRecord || rec instanceof PageSnapshot || rec instanceof MemoryRecoveryRecord)) + return null; + FileWriteHandle currWrHandle = currentHandle(); WALDisableContext isDisable = walDisableContext; @@ -1773,7 +1793,7 @@ private void shutdown() throws IgniteInterruptedCheckedException { blockingSectionEnd(); } - if (evt.isRecordable(EVT_WAL_SEGMENT_ARCHIVED)) { + if (evt.isRecordable(EVT_WAL_SEGMENT_ARCHIVED) && !cctx.kernalContext().recoveryMode()) { evt.record(new WalSegmentArchivedEvent( cctx.discovery().localNode(), res.getAbsIdx(), @@ -2012,6 +2032,8 @@ private void shutdown() throws IgniteInterruptedCheckedException { for (FileCompressorWorker worker: workers) U.join(worker); + workers.clear(); + U.cancel(this); } @@ -2085,7 +2107,7 @@ private void body0() { f0.force(); } - if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED)) { + if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED) && !cctx.kernalContext().recoveryMode()) { evt.record(new WalSegmentCompactedEvent( cctx.localNode(), segIdx, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index def9bf20a7c3b..c74c7eff8d34e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -76,9 +76,12 @@ import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord; +import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; +import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext; @@ -442,6 +445,20 @@ public void setFileIOFactory(FileIOFactory ioFactory) { } } + /** + * + */ + private void startArchiverAndCompressor() { + if (isArchiverEnabled()) { + assert archiver != null; + + new IgniteThread(archiver).start(); + } + + if (compressor != null) + compressor.start(); + } + /** * @throws IgniteCheckedException if WAL store path is configured and archive path isn't (or vice versa) */ @@ -481,11 +498,17 @@ private void checkWalConfiguration() throws IgniteCheckedException { if (archiver != null) archiver.shutdown(); - if (compressor != null) + if (compressor != null) { compressor.shutdown(); - if (decompressor != null) + compressor = null; + } + + if (decompressor != null) { decompressor.shutdown(); + + decompressor = null; + } } catch (Exception e) { U.error(log, "Failed to gracefully close WAL segment: " + currentHnd.fileIO, e); @@ -499,17 +522,6 @@ private void checkWalConfiguration() throws IgniteCheckedException { " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); start0(); - - if (!cctx.kernalContext().clientNode()) { - if (isArchiverEnabled()) { - assert archiver != null; - - new IgniteThread(archiver).start(); - } - - if (compressor != null) - compressor.start(); - } } /** {@inheritDoc} */ @@ -537,6 +549,9 @@ private void checkWalConfiguration() throws IgniteCheckedException { @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException { assert currentHnd == null; assert lastPtr == null || lastPtr instanceof FileWALPointer; + + startArchiverAndCompressor(); + assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) : "Trying to restore FileWriteHandle on deactivated write ahead log manager"; @@ -701,6 +716,11 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { if (serializer == null || mode == WALMode.NONE) return null; + // Only delta-records, page snapshots and memory recovery are allowed to write in recovery mode. + if (cctx.kernalContext().recoveryMode() && + !(record instanceof PageDeltaRecord || record instanceof PageSnapshot || record instanceof MemoryRecoveryRecord)) + return null; + FileWriteHandle currWrHandle = currentHandle(); WALDisableContext isDisable = walDisableContext; @@ -1649,7 +1669,7 @@ private synchronized void release(long absIdx) { notifyAll(); } - if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED)) { + if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED) && !cctx.kernalContext().recoveryMode()) { evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(), res.getAbsIdx(), res.getDstArchiveFile())); } @@ -2015,7 +2035,7 @@ private void deleteObsoleteRawSegments() { f0.force(); } - if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED)) { + if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED) && !cctx.kernalContext().recoveryMode()) { evt.record(new WalSegmentCompactedEvent( cctx.discovery().localNode(), currReservedSegment, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index 6d379a9381b15..e70a02709f4f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -652,6 +652,11 @@ protected IgniteConfiguration prepareIgniteConfiguration() { return null; } + /** {@inheritDoc} */ + @Override public boolean recoveryMode() { + return false; + } + /** {@inheritDoc} */ @Override public PdsFoldersResolver pdsFolderResolver() { return new PdsFoldersResolver() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 2c8f03fc9dde0..ee40039994548 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -221,7 +221,7 @@ public RecordDataV1Serializer(GridCacheSharedContext cctx) { * @return {@code True} if this record should be encrypted. */ private boolean needEncryption(WALRecord rec) { - if (!(rec instanceof WalRecordCacheGroupAware)) + if (!(rec instanceof WalRecordCacheGroupAware) || rec instanceof MetastoreDataRecord) return false; return needEncryption(((WalRecordCacheGroupAware)rec).groupId()); @@ -1925,7 +1925,7 @@ RecordType recordType(WALRecord rec) { */ boolean isDataRecordEncrypted(DataRecord rec) { for (DataEntry e : rec.writeEntries()) { - if(needEncryption(cctx.cacheContext(e.cacheId()).groupId())) + if (cctx.cacheContext(e.cacheId()) != null && needEncryption(cctx.cacheContext(e.cacheId()).groupId())) return true; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java index 9e2286e43fe8c..aa53c39bd6693 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -1379,7 +1379,7 @@ final void checkNoCaches(int nodes) { GridCacheProcessor cache = ((IgniteEx)ignite(i)).context().cache(); assertTrue(cache.caches().isEmpty()); - assertTrue(cache.internalCaches().isEmpty()); + assertTrue(cache.internalCaches().stream().allMatch(c -> c.context().isRecoveryMode())); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java index 28c1e9b2a5838..38744b0b182ad 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java @@ -74,6 +74,8 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); + cfg.setConsistentId(gridName); + if (customWorkSubDir != null) cfg.setWorkDirectory(Paths.get(U.defaultWorkDirectory(), customWorkSubDir).toString()); @@ -358,8 +360,8 @@ private void copyIncompatibleBinaryMetadata(String fromWorkDir, ) throws Exception { String workDir = U.defaultWorkDirectory(); - Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", "node00-" + fromConsId, fileName); - Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", "node00-" + toConsId, fileName); + Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", fromConsId, fileName); + Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", toConsId, fileName); Files.copy(fromFile, toFile, StandardCopyOption.REPLACE_EXISTING); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java index 14d0fb6d4dfb7..da19285d6301d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java @@ -317,16 +317,6 @@ private static boolean isPartitionFile(File file) { return file.getName().contains("part") && file.getName().endsWith("bin"); } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - FileIO delegate = delegateFactory.create(file); - - if (isPartitionFile(file)) - return new HaltOnTruncateFileIO(delegate, file); - - return delegate; - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { FileIO delegate = delegateFactory.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java index 059b5eefdcf73..39faf5a1179ff 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java @@ -54,9 +54,6 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID; @@ -180,6 +177,9 @@ public void testNodeInvalidatedWhenPersistenceIsCorrupted() throws Exception { startGrid(0); } catch (IgniteCheckedException ex) { + if (X.hasCause(ex, StorageException.class, IOException.class)) + return; // Success; + throw ex; } @@ -458,11 +458,6 @@ private static class FailingFileIOFactory implements FileIOFactory { /** Create FileIO closure. */ private volatile IgniteBiClosure createClo; - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... openOption) throws IOException { FileIO fileIO = null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java index 3605700d739fc..578692c97e840 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java @@ -449,16 +449,6 @@ private static boolean isPartitionFile(File file) { return file.getName().contains("part") && file.getName().endsWith("bin"); } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - FileIO delegate = delegateFactory.create(file); - - if (isPartitionFile(file)) - return new FailingFileIO(delegate); - - return delegate; - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { FileIO delegate = delegateFactory.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java index b36bac0123d6e..6b457ef47196f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java @@ -287,11 +287,6 @@ private static class SlowIOFactory implements FileIOFactory { /** */ private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... openOption) throws IOException { final FileIO delegate = delegateFactory.create(file, openOption); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java index 4dd5f51b28772..48b60d48c9981 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java @@ -545,11 +545,6 @@ private static class TestFileIOFactory implements FileIOFactory { this.delegate = delegate; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return new TestFileIO(delegate.create(file)); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { return new TestFileIO(delegate.create(file, modes)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java index 3afafe691f462..f58d02d93e944 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java @@ -335,11 +335,6 @@ private static class SlowCheckpointFileIOFactory implements FileIOFactory { /** Delegate factory. */ private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... openOption) throws IOException { final FileIO delegate = delegateFactory.create(file, openOption); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java new file mode 100644 index 0000000000000..584196f6797b2 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java @@ -0,0 +1,577 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.OpenOption; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; +import com.google.common.collect.Lists; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.failure.FailureHandler; +import org.apache.ignite.failure.StopNodeFailureHandler; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.wal.memtracker.PageMemoryTrackerConfiguration; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; + +/** + * A set of tests that check correctness of logical recovery performed during node start. + */ +public class IgniteLogicalRecoveryTest extends GridCommonAbstractTest { + /** Shared group name. */ + private static final String SHARED_GROUP_NAME = "group"; + + /** Dynamic cache prefix. */ + private static final String DYNAMIC_CACHE_PREFIX = "dynamic-cache-"; + + /** Cache prefix. */ + private static final String CACHE_PREFIX = "cache-"; + + /** Io factory. */ + private FileIOFactory ioFactory; + + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setConsistentId(igniteInstanceName); + + cfg.setCacheConfiguration( + cacheConfiguration(CACHE_PREFIX + 0, CacheMode.PARTITIONED, CacheAtomicityMode.ATOMIC), + cacheConfiguration(CACHE_PREFIX + 1, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL), + cacheConfiguration(CACHE_PREFIX + 2, CacheMode.REPLICATED, CacheAtomicityMode.ATOMIC), + cacheConfiguration(CACHE_PREFIX + 3, CacheMode.REPLICATED, CacheAtomicityMode.TRANSACTIONAL), + cacheConfiguration(CACHE_PREFIX + 4, SHARED_GROUP_NAME, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL), + cacheConfiguration(CACHE_PREFIX + 5, SHARED_GROUP_NAME, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL) + ); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setAlwaysWriteFullPages(true) + .setWalMode(WALMode.LOG_ONLY) + .setCheckpointFrequency(1024 * 1024 * 1024) // Disable automatic checkpoints. + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setName("dflt") + .setInitialSize(256 * 1024 * 1024) + .setMaxSize(256 * 1024 * 1024) + .setPersistenceEnabled(true) + ); + + cfg.setDataStorageConfiguration(dsCfg); + + if (ioFactory != null) + dsCfg.setFileIOFactory(ioFactory); + + cfg.setPluginConfigurations(new PageMemoryTrackerConfiguration().setEnabled(false).setCheckPagesOnCheckpoint(true)); + + return cfg; + } + + /** + * @param name Name. + * @param cacheMode Cache mode. + * @param atomicityMode Atomicity mode. + */ + private CacheConfiguration cacheConfiguration(String name, CacheMode cacheMode, CacheAtomicityMode atomicityMode) { + return cacheConfiguration(name, null, cacheMode, atomicityMode); + } + + /** + * @param name Name. + * @param groupName Group name. + * @param cacheMode Cache mode. + * @param atomicityMode Atomicity mode. + */ + private CacheConfiguration cacheConfiguration(String name, @Nullable String groupName, CacheMode cacheMode, CacheAtomicityMode atomicityMode) { + CacheConfiguration cfg = new CacheConfiguration<>(); + + cfg.setGroupName(groupName); + cfg.setName(name); + cfg.setCacheMode(cacheMode); + cfg.setAtomicityMode(atomicityMode); + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cfg.setBackups(2); + cfg.setAffinity(new RendezvousAffinityFunction(false, 32)); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, "true"); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP); + } + + /** + * + */ + public void testRecoveryOnJoinToActiveCluster() throws Exception { + IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3); + + crd.cluster().active(true); + + IgniteEx node = grid(2); + + AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); + + aggCacheLoader.start(); + + U.sleep(3000); + + forceCheckpoint(); + + U.sleep(3000); + + aggCacheLoader.stop(); + + stopGrid(2, true); + + startGrid(2); + + awaitPartitionMapExchange(); + + aggCacheLoader.consistencyCheck(grid(2)); + } + + /** + * + */ + public void testRecoveryOnJoinToInactiveCluster() throws Exception { + IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3); + + crd.cluster().active(true); + + IgniteEx node = grid(2); + + AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); + + aggCacheLoader.start(); + + U.sleep(3000); + + forceCheckpoint(); + + U.sleep(3000); + + aggCacheLoader.stop(); + + stopGrid(2, true); + + crd.cluster().active(false); + + startGrid(2); + + crd.cluster().active(true); + + awaitPartitionMapExchange(); + + aggCacheLoader.consistencyCheck(grid(2)); + } + + /** + * + */ + public void testRecoveryOnDynamicallyStartedCaches() throws Exception { + List dynamicCaches = Lists.newArrayList( + cacheConfiguration(DYNAMIC_CACHE_PREFIX + 0, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL), + cacheConfiguration(DYNAMIC_CACHE_PREFIX + 1, CacheMode.REPLICATED, CacheAtomicityMode.TRANSACTIONAL), + cacheConfiguration(DYNAMIC_CACHE_PREFIX + 2, CacheMode.PARTITIONED, CacheAtomicityMode.ATOMIC), + cacheConfiguration(DYNAMIC_CACHE_PREFIX + 3, CacheMode.REPLICATED, CacheAtomicityMode.ATOMIC) + ); + + doTestWithDynamicCaches(dynamicCaches); + } + + /** + * + */ + public void testRecoveryWithMvccCaches() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10052"); + + List dynamicCaches = Lists.newArrayList( + cacheConfiguration(DYNAMIC_CACHE_PREFIX + 0, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT), + cacheConfiguration(DYNAMIC_CACHE_PREFIX + 1, CacheMode.REPLICATED, CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) + ); + + doTestWithDynamicCaches(dynamicCaches); + } + + /** + * @param dynamicCaches Dynamic caches. + */ + private void doTestWithDynamicCaches(List dynamicCaches) throws Exception { + IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3); + + crd.cluster().active(true); + + IgniteEx node = grid(2); + + node.getOrCreateCaches(dynamicCaches); + + AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); + + aggCacheLoader.start(); + + U.sleep(3000); + + forceCheckpoint(); + + U.sleep(3000); + + aggCacheLoader.stop(); + + stopGrid(2, true); + + startGrid(2); + + awaitPartitionMapExchange(); + + for (int idx = 0; idx < 3; idx++) + aggCacheLoader.consistencyCheck(grid(idx)); + } + + /** + * + */ + public void testRecoveryOnJoinToDifferentBlt() throws Exception { + IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3); + + crd.cluster().active(true); + + IgniteEx node = grid(2); + + AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); + + aggCacheLoader.start(); + + U.sleep(3000); + + forceCheckpoint(); + + U.sleep(3000); + + aggCacheLoader.stop(); + + stopGrid(2, true); + + resetBaselineTopology(); + + startGrid(2); + + resetBaselineTopology(); + + awaitPartitionMapExchange(); + + for (int idx = 0; idx < 3; idx++) + aggCacheLoader.consistencyCheck(grid(idx)); + } + + /** + * + */ + public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { + // Crash recovery fails because of the bug in pages recycling. + // Test passes if don't perform removes in cache loader. + fail("https://issues.apache.org/jira/browse/IGNITE-9303"); + + IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3, false); + + crd.cluster().active(true); + + IgniteEx node = grid(2); + + AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); + + aggCacheLoader.start(); + + U.sleep(3000); + + forceCheckpoint(); + + U.sleep(3000); + + aggCacheLoader.stop(); + + stopGrid(2, false); + + ioFactory = new CheckpointFailIoFactory(); + + IgniteInternalFuture startNodeFut = GridTestUtils.runAsync(() -> startGrid(2)); + + try { + startNodeFut.get(); + } + catch (Exception expected) { } + + ioFactory = null; + + // Start node again and check recovery. + startGrid(2); + + awaitPartitionMapExchange(); + + for (int idx = 0; idx < 3; idx++) + aggCacheLoader.consistencyCheck(grid(idx)); + } + + /** {@inheritDoc} */ + @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { + return new StopNodeFailureHandler(); + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return 600 * 1000; + } + + /** + * + */ + private class AggregateCacheLoader { + /** Ignite. */ + IgniteEx ignite; + + /** Stop flag. */ + AtomicBoolean stopFlag; + + /** Cache loaders. */ + Map cacheLoaders; + + /** + * @param ignite Ignite. + */ + public AggregateCacheLoader(IgniteEx ignite) { + this.ignite = ignite; + } + + /** + * + */ + public void start() { + if (stopFlag != null && !stopFlag.get()) + throw new IllegalStateException("Cache loaders must be stopped before start again"); + + stopFlag = new AtomicBoolean(); + cacheLoaders = new HashMap<>(); + + for (String cacheName : ignite.cacheNames()) { + CacheLoader loader = new CacheLoader(ignite, stopFlag, cacheName); + + IgniteInternalFuture loadFuture = GridTestUtils.runAsync(loader); + + cacheLoaders.put(loader, loadFuture); + } + } + + /** + * + */ + public void stop() throws IgniteCheckedException { + if (stopFlag != null) + stopFlag.set(true); + + if (cacheLoaders != null) + for (IgniteInternalFuture loadFuture : cacheLoaders.values()) + loadFuture.get(); + } + + /** + * @param ignite Ignite. + */ + public void consistencyCheck(IgniteEx ignite) throws IgniteCheckedException { + if (cacheLoaders != null) + for (CacheLoader cacheLoader : cacheLoaders.keySet()) + cacheLoader.consistencyCheck(ignite); + } + } + + /** + * + */ + static class CacheLoader implements Runnable { + /** Keys space. */ + static final int KEYS_SPACE = 3096; + + /** Ignite. */ + final IgniteEx ignite; + + /** Stop flag. */ + final AtomicBoolean stopFlag; + + /** Cache name. */ + final String cacheName; + + /** Local cache. */ + final Map locCache = new ConcurrentHashMap<>(); + + /** + * @param ignite Ignite. + * @param stopFlag Stop flag. + * @param cacheName Cache name. + */ + public CacheLoader(IgniteEx ignite, AtomicBoolean stopFlag, String cacheName) { + this.ignite = ignite; + this.stopFlag = stopFlag; + this.cacheName = cacheName; + } + + /** {@inheritDoc} */ + @Override public void run() { + while (!stopFlag.get()) { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + int key = rnd.nextInt(KEYS_SPACE); + + boolean remove = rnd.nextInt(100) <= 20; + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + + try { + if (remove) { + cache.remove(key); + + locCache.remove(key); + } + else { + int[] payload = new int[KEYS_SPACE]; + Arrays.fill(payload, key); + + cache.put(key, payload); + + locCache.put(key, payload); + } + } + catch (Exception ignored) { } + } + } + + /** + * + */ + public void consistencyCheck(IgniteEx ignite) { + IgniteCache cache = ignite.getOrCreateCache(cacheName); + + for (int key = 0; key < KEYS_SPACE; key++) { + int[] expectedValue = (int[]) locCache.get(key); + int[] actualValue = (int[]) cache.get(key); + + Assert.assertEquals("Consistency check failed for: " + cache.getName() + ", key=" + key, + arrayToString(expectedValue), arrayToString(actualValue)); + } + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + CacheLoader loader = (CacheLoader) o; + + return Objects.equals(cacheName, loader.cacheName); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(cacheName); + } + } + + /** + * + */ + static class CheckpointFailIoFactory implements FileIOFactory { + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... modes) throws IOException { + FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); + + if (file.getName().contains("part-")) + return new FileIODecorator(delegate) { + @Override public int write(ByteBuffer srcBuf) throws IOException { + throw new IOException("test"); + } + + @Override public int write(ByteBuffer srcBuf, long position) throws IOException { + throw new IOException("test"); + } + + @Override public int write(byte[] buf, int off, int len) throws IOException { + throw new IOException("test"); + } + }; + + return delegate; + } + } + + /** + * @param arr Array. + */ + static String arrayToString(int[] arr) { + if (arr == null) + return "null"; + + StringBuilder sb = new StringBuilder(); + sb.append('['); + for (int i = 0; i < Math.min(arr.length, 10); i++) + sb.append(i + ","); + sb.append(']'); + + return sb.toString(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java index 4a22a2bcd7f1d..c05f65c26b6b0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java @@ -323,7 +323,8 @@ private void checkMetricsConsistency(final IgniteEx node, String cacheName) thro .memoryMetrics() .getTotalAllocatedPages(); - assertEquals(totalPersistenceSize / pageStoreMgr.pageSize(), totalAllocatedPagesFromMetrics); + assertEquals("Number of allocated pages is different than in metrics for [node=" + node.name() + ", cache=" + cacheName + "]", + totalPersistenceSize / pageStoreMgr.pageSize(), totalAllocatedPagesFromMetrics); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java index a214fc3fbd1b5..43302855dc541 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.failure.StopNodeFailureHandler; import org.apache.ignite.internal.GridKernalState; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; @@ -51,9 +52,6 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Assert; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP; /** @@ -96,6 +94,8 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + cfg.setConsistentId(igniteInstanceName); + DataStorageConfiguration dsCfg = new DataStorageConfiguration() .setDefaultDataRegionConfiguration( new DataRegionConfiguration().setMaxSize(100L * 1024 * 1024).setPersistenceEnabled(true)) @@ -128,27 +128,36 @@ public void testRecoveringOnCacheInitFail() throws Exception { // Fail to initialize page store. 2 extra pages is needed for MetaStorage. ioFactory = new FilteringFileIOFactory(".bin", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 2 * PAGE_SIZE)); - final IgniteEx grid = startGrid(0); - boolean failed = false; + + IgniteInternalFuture startGridFut = GridTestUtils.runAsync(() -> { + try { + IgniteEx grid = startGrid(0); + + grid.cluster().active(true); + } + catch (Exception e) { + throw new RuntimeException("Failed to start node.", e); + } + }); + try { - grid.cluster().active(true); - } catch (Exception expected) { - log.warning("Expected cache error", expected); + startGridFut.get(); + } + catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Failed to start node.")); failed = true; } Assert.assertTrue("Cache initialization must failed", failed); - // Grid should be automatically stopped after checkpoint fail. - awaitStop(grid); - // Grid should be successfully recovered after stopping. ioFactory = null; - IgniteEx recoveredGrid = startGrid(0); - recoveredGrid.active(true); + IgniteEx grid = startGrid(0); + + grid.cluster().active(true); } /** @@ -316,7 +325,9 @@ public void testRecoveringOnCheckpointWriteFail() throws Exception { public void testRecoveringOnWALWritingFail1() throws Exception { // Allow to allocate only 1 wal segment, fail on write to second. ioFactory = new FilteringFileIOFactory(".wal", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), WAL_SEGMENT_SIZE)); + System.setProperty(IGNITE_WAL_MMAP, "true"); + doTestRecoveringOnWALWritingFail(); } @@ -326,7 +337,9 @@ public void testRecoveringOnWALWritingFail1() throws Exception { public void testRecoveringOnWALWritingFail2() throws Exception { // Fail somewhere on the second wal segment. ioFactory = new FilteringFileIOFactory(".wal", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), (long) (1.5 * WAL_SEGMENT_SIZE))); + System.setProperty(IGNITE_WAL_MMAP, "false"); + doTestRecoveringOnWALWritingFail(); } @@ -334,18 +347,23 @@ public void testRecoveringOnWALWritingFail2() throws Exception { * Test node stopping & recovery on WAL writing fail. */ private void doTestRecoveringOnWALWritingFail() throws Exception { - final IgniteEx grid = startGrid(0); + IgniteEx grid = startGrid(0); FileWriteAheadLogManager wal = (FileWriteAheadLogManager)grid.context().cache().context().wal(); + wal.setFileIOFactory(ioFactory); grid.cluster().active(true); int failedPosition = -1; - for (int i = 0; i < 1000; i++) { + final int keysCount = 2000; + + final int dataSize = 2048; + + for (int i = 0; i < keysCount; i++) { byte payload = (byte) i; - byte[] data = new byte[2048]; + byte[] data = new byte[dataSize]; Arrays.fill(data, payload); try { @@ -359,7 +377,7 @@ private void doTestRecoveringOnWALWritingFail() throws Exception { } // We must be able to put something into cache before fail. - Assert.assertTrue(failedPosition > 0); + Assert.assertTrue("One of the cache puts must be failed", failedPosition > 0); // Grid should be automatically stopped after WAL fail. awaitStop(grid); @@ -367,15 +385,16 @@ private void doTestRecoveringOnWALWritingFail() throws Exception { ioFactory = null; // Grid should be successfully recovered after stopping. - IgniteEx recoveredGrid = startGrid(0); - recoveredGrid.cluster().active(true); + grid = startGrid(0); + + grid.cluster().active(true); for (int i = 0; i < failedPosition; i++) { byte payload = (byte) i; - byte[] data = new byte[2048]; + byte[] data = new byte[dataSize]; Arrays.fill(data, payload); - byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i); + byte[] actualData = (byte[]) grid.cache(CACHE_NAME).get(i); Assert.assertArrayEquals(data, actualData); } } @@ -474,11 +493,6 @@ private static class FilteringFileIOFactory implements FileIOFactory { this.pattern = pattern; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, WRITE, READ); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { if (file.getName().endsWith(pattern)) @@ -509,11 +523,6 @@ private LimitedSizeFileIOFactory(FileIOFactory delegate, long fsSpaceBytes) { this.availableSpaceBytes = new AtomicLong(fsSpaceBytes); } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return new LimitedSizeFileIO(delegate.create(file), availableSpaceBytes); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { return new LimitedSizeFileIO(delegate.create(file, modes), availableSpaceBytes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java index a744ab10c1671..906f19199a4b7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java @@ -48,7 +48,6 @@ import static java.nio.file.FileVisitResult.CONTINUE; import static java.nio.file.Files.walkFileTree; import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.CP_FILE_NAME_PATTERN; - import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.META_STORAGE_NAME; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX; @@ -92,6 +91,8 @@ public class IgniteNodeStoppedDuringDisableWALTest extends GridCommonAbstractTes } /** + * Test checks that after WAL is globally disabled and node is stopped, persistent store is cleaned properly after node restart. + * * @throws Exception If failed. */ public void test() throws Exception { @@ -190,6 +191,8 @@ private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exce String msg = nodeStopPoint.toString(); + int pageSize = ig1.configuration().getDataStorageConfiguration().getPageSize(); + if (nodeStopPoint.needCleanUp) { PdsFoldersResolver foldersResolver = ((IgniteEx)ig1).context().pdsFolderResolver(); @@ -215,14 +218,14 @@ private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exce if (CP_FILE_NAME_PATTERN.matcher(name).matches()) failed = true; - if (name.startsWith(PART_FILE_PREFIX)) + if (name.startsWith(PART_FILE_PREFIX) && path.toFile().length() > pageSize) failed = true; - if (name.startsWith(INDEX_FILE_NAME)) + if (name.startsWith(INDEX_FILE_NAME) && path.toFile().length() > pageSize) failed = true; if (failed) - fail(msg + " " + filePath); + fail(msg + " " + filePath + " " + path.toFile().length()); return CONTINUE; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java index 351a42cb5eb66..6d6baddf8f8ba 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java @@ -185,11 +185,6 @@ private static class FailingFileIOFactory implements FileIOFactory { this.fail = fail; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { final FileIO delegate = delegateFactory.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java index a28ec5fe1e74c..0134e6974291f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java @@ -256,11 +256,6 @@ private static class FailingFileIOFactory implements FileIOFactory { this.fail = fail; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { final FileIO delegate = delegateFactory.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java index 5a1a6fa179654..a77ab989a939b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java @@ -112,6 +112,8 @@ public void testNodeStartFailedFsync() throws Exception { * @throws Exception If failed. */ public void testFailureHandlerTriggeredFsync() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10035"); + fsync = true; failFormatFileOnClusterActivate(); @@ -121,6 +123,8 @@ public void testFailureHandlerTriggeredFsync() throws Exception { * @throws Exception If failed. */ public void testFailureHandlerTriggered() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10035"); + fsync = false; failFormatFileOnClusterActivate(); @@ -224,11 +228,6 @@ private static class FailingFileIOFactory implements FileIOFactory { this.failMtdNameRef = failMtdNameRef; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { final FileIO delegate = delegateFactory.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java index 6d2b1f79227b5..4c34093a10544 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java @@ -114,7 +114,7 @@ public void testReservedOnExchange() throws Exception { final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt + 1); - ig0.active(true); + ig0.cluster().active(true); stopGrid(initGridCnt); @@ -256,7 +256,7 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { IgniteEx ig0 = (IgniteEx)startGrids(2); - ig0.active(true); + ig0.cluster().active(true); IgniteCache cache = ig0.cache("cache1"); @@ -270,6 +270,8 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { IgniteEx ig1 = startGrid(1); + awaitPartitionMapExchange(); + IgniteCache cache1 = ig1.cache("cache1"); assertEquals(entryCnt / 2, cache.size()); @@ -286,8 +288,6 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { } } - cache.rebalance().get(); - for (int p = 0; p < ig1.affinity("cache1").partitions(); p++) { GridDhtLocalPartition p0 = ig0.context().cache().cache("cache1").context().topology().localPartition(p); GridDhtLocalPartition p1 = ig1.context().cache().cache("cache1").context().topology().localPartition(p); @@ -305,7 +305,7 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception { IgniteEx ig0 = (IgniteEx)startGrids(2); - ig0.active(true); + ig0.cluster().active(true); IgniteCache cache = ig0.cache("cache1"); @@ -330,6 +330,8 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception { IgniteEx ig1 = startGrid(1); + awaitPartitionMapExchange(); + IgniteCache cache1 = ig1.cache("cache1"); assertEquals(entryCnt / 2, cache.size()); @@ -346,8 +348,6 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception { } } - cache.rebalance().get(); - for (int p = 0; p < ig1.affinity("cache1").partitions(); p++) { GridDhtLocalPartition p0 = ig0.context().cache().cache("cache1").context().topology().localPartition(p); GridDhtLocalPartition p1 = ig1.context().cache().cache("cache1").context().topology().localPartition(p); @@ -407,7 +407,7 @@ public void testNodeLeftDuringExchange() throws Exception { final Ignite ig0 = startGrids(initGridCnt); - ig0.active(true); + ig0.cluster().active(true); IgniteCache cache = ig0.cache("cache1"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java index 57565bfd58c48..0409a297be1b1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java @@ -612,11 +612,6 @@ static class FailingIOFactory implements FileIOFactory { this.delegate = delegate; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, WRITE, READ); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { FileIO delegateIO = delegate.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java index 261167a7d6170..d2ea4f114d372 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java @@ -49,8 +49,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager; import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; @@ -584,6 +584,8 @@ public void testRecoveryNoPageLost2() throws Exception { } /** + * Test checks that the number of pages per each page store are not changing before and after node restart. + * * @throws Exception If failed. */ public void testRecoveryNoPageLost3() throws Exception { @@ -679,7 +681,7 @@ private void recoveryNoPageLost(boolean checkpoint) throws Exception { pages = allocatedPages(ignite, CACHE2_NAME); - ignite.close(); + stopGrid(0, true); } } finally { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java index 7ff134850387b..b05d5db75a197 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java @@ -287,11 +287,6 @@ private static class SlowCheckpointFileIOFactory implements FileIOFactory { /** Delegate factory. */ private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... openOption) throws IOException { final FileIO delegate = delegateFactory.create(file, openOption); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java index 219db8d166433..a888ecb2607c8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java @@ -786,7 +786,9 @@ public Runnable checkCacheEmpty() { Map caches = caches(ig); - Assert.assertEquals(0, caches.size()); + for (GridCacheAdapter cacheAdapter : caches.values()) + Assert.assertTrue("Cache should be in recovery mode: " + cacheAdapter.context(), + cacheAdapter.context().isRecoveryMode()); } }); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalDeltaConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalDeltaConsistencyTest.java index 7f872fd005cb7..e0057b0c75ed1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalDeltaConsistencyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalDeltaConsistencyTest.java @@ -76,6 +76,8 @@ protected boolean checkPagesOnCheckpoint() { protected DataStorageConfiguration getDataStorageConfiguration() { return new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setInitialSize(256 * 1024 * 1024) + .setMaxSize(256 * 1024 * 1024) .setPersistenceEnabled(true) .setName("dflt-plc")); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java index 1b9a18a8de1e6..e9512e111ec0f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java @@ -72,6 +72,8 @@ public final void testNotEmptyPds() throws Exception { for (int i = 0; i < 3_000; i++) cache.put(i, "Cache value " + i); + forceCheckpoint(); + stopGrid(0); ignite = startGrid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java index 09cacfbb2c5cc..864d2cc0137e8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java @@ -475,18 +475,7 @@ else if (record instanceof InitNewPageRecord) return; // Increment statistics. - AtomicInteger statCnt = stats.get(record.type()); - - if (statCnt == null) { - statCnt = new AtomicInteger(); - - AtomicInteger oldCnt = stats.putIfAbsent(record.type(), statCnt); - - if (oldCnt != null) - statCnt = oldCnt; - } - - statCnt.incrementAndGet(); + stats.computeIfAbsent(record.type(), r -> new AtomicInteger()).incrementAndGet(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTrackerPluginProvider.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTrackerPluginProvider.java index c5f83b5ef68dd..cad3b0269dd75 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTrackerPluginProvider.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTrackerPluginProvider.java @@ -20,12 +20,16 @@ import java.io.Serializable; import java.util.UUID; import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.plugin.CachePluginContext; @@ -43,7 +47,7 @@ * PageMemory tracker plugin provider. */ public class PageMemoryTrackerPluginProvider implements PluginProvider, - IgniteChangeGlobalStateSupport { + IgniteChangeGlobalStateSupport, DatabaseLifecycleListener { /** System property name to implicitly enable page memory tracker . */ public static final String IGNITE_ENABLE_PAGE_MEMORY_TRACKER = "IGNITE_ENABLE_PAGE_MEMORY_TRACKER"; @@ -132,7 +136,7 @@ else if (IgnitePageStoreManager.class.equals(cls)) /** {@inheritDoc} */ @Override public void start(PluginContext ctx) { - // No-op + ((IgniteEx)ctx.grid()).context().internalSubscriptionProcessor().registerDatabaseListener(this); } /** {@inheritDoc} */ @@ -197,4 +201,15 @@ public static PageMemoryTracker tracker(Ignite ignite) { return null; } } + + @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + if (plugin != null) { + try { + plugin.start(); + } + catch (Exception e) { + log.error("Can't start plugin", e); + } + } + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java index 4d2bdcfcc82fb..6aa2a33e77142 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java @@ -161,11 +161,6 @@ private String getArchiveWalDirPath(Ignite ignite) throws IgniteCheckedException * */ private static class CountedFileIOFactory extends RandomAccessFileIOFactory { - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { return new CountedFileIO(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java index 473eaf5ce97bc..fd31ee5e78f56 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java @@ -181,9 +181,6 @@ class IndexCorruptionFailoverScenario implements FailoverScenario { return hnd; if (treeCorruptionPred.apply(hnd, tree)) { - log.info("Created corrupted tree handler [nodeOrder=" + locIgnite.localNode().order() + ", hnd=" + hnd + - ", tree=" + tree + ']'); - PageHandler delegate = (PageHandler)hnd; return new PageHandler() { diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java index c8cf9aae7bb08..4159f6ed742a4 100644 --- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java +++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java @@ -90,6 +90,7 @@ public GridCacheTestContext(GridTestKernalContext ctx) throws Exception { AffinityTopologyVersion.ZERO, true, true, + false, new GridCacheEventManager(), new CacheOsStoreManager(null, new CacheConfiguration()), new GridCacheEvictionManager(), diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index f1d66825b715b..0195a812d6f9d 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -182,9 +182,6 @@ public abstract class GridAbstractTest extends TestCase { /** Timestamp for tests. */ private static long ts = System.currentTimeMillis(); - /** Starting Ignite instance name. */ - protected static final ThreadLocal startingIgniteInstanceName = new ThreadLocal<>(); - /** Force failure flag. */ private boolean forceFailure; @@ -890,7 +887,7 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G checkConfiguration(cfg); if (!isRemoteJvm(igniteInstanceName)) { - startingIgniteInstanceName.set(igniteInstanceName); + IgniteUtils.setCurrentIgniteName(igniteInstanceName); try { String cfgProcClsName = System.getProperty(IGNITE_CFG_PREPROCESSOR_CLS); @@ -926,7 +923,7 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G return node; } finally { - startingIgniteInstanceName.set(null); + IgniteUtils.setCurrentIgniteName(null); } } else diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java index 8a28e9eac1c47..ea555a0c583f8 100644 --- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java +++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java @@ -144,11 +144,6 @@ public AlignedBuffersDirectFileIOFactory( return allocate; } - /** {@inheritDoc} */ - @Override public FileIO create(File file) throws IOException { - return create(file, CREATE, READ, WRITE); - } - /** {@inheritDoc} */ @Override public FileIO create(File file, OpenOption... modes) throws IOException { if (useBackupFactory) diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java index e9159bf7cd553..214a541a17214 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxHistoricalRebalancingTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest; import org.apache.ignite.internal.processors.cache.persistence.PersistenceDirectoryWarningLoggingTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgniteLogicalRecoveryTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheDestroyDuringCheckpointTest; @@ -88,6 +89,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgnitePdsCorruptedIndexTest.class); + suite.addTestSuite(IgniteLogicalRecoveryTest.class); + return suite; } } From caaa423b5da3744bc4b7073eb2f464ff471e4845 Mon Sep 17 00:00:00 2001 From: Stanislav Lukyanov Date: Fri, 2 Nov 2018 11:49:00 +0300 Subject: [PATCH 067/403] IGNITE-9841: SQL: Improved tracking of LOST partitions for queries when persistence is enabled. This closes #5069. --- ...gniteCachePartitionLossPolicySelfTest.java | 435 ++++++++++++++++-- .../h2/twostep/GridReduceQueryExecutor.java | 21 + ...exingCachePartitionLossPolicySelfTest.java | 87 +--- 3 files changed, 428 insertions(+), 115 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java index 1616e8f41359d..caf0829f0417d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java @@ -24,17 +24,23 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import javax.cache.CacheException; +import junit.framework.AssertionFailedError; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.PartitionLossPolicy; import org.apache.ignite.cache.affinity.Affinity; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.CacheRebalancingEvent; import org.apache.ignite.events.Event; @@ -81,7 +87,10 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe private final AtomicBoolean delayPartExchange = new AtomicBoolean(false); /** */ - private final TopologyChanger killSingleNode = new TopologyChanger(false, Arrays.asList(3), Arrays.asList(0, 1, 2, 4),0); + private final TopologyChanger killSingleNode = new TopologyChanger(false, Collections.singletonList(3), Arrays.asList(0, 1, 2, 4), 0); + + /** */ + private boolean isPersistenceEnabled; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -103,6 +112,10 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe cfg.setCacheConfiguration(cacheConfiguration()); + cfg.setDataStorageConfiguration(new DataStorageConfiguration().setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setPersistenceEnabled(isPersistenceEnabled) + )); + return cfg; } @@ -121,18 +134,26 @@ protected CacheConfiguration cacheConfiguration() { return cacheCfg; } - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopAllGrids(); - } - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); delayPartExchange.set(false); + partLossPlc = PartitionLossPolicy.IGNORE; + backups = 0; + + isPersistenceEnabled = false; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); } /** @@ -144,6 +165,17 @@ public void testReadOnlySafe() throws Exception { checkLostPartition(false, true, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadOnlySafeWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(false, true, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -153,6 +185,19 @@ public void testReadOnlyAll() throws Exception { checkLostPartition(false, false, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadOnlyAllWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.READ_ONLY_ALL; + + isPersistenceEnabled = true; + + checkLostPartition(false, false, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -162,6 +207,17 @@ public void testReadWriteSafe() throws Exception { checkLostPartition(true, true, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -171,6 +227,19 @@ public void testReadWriteAll() throws Exception { checkLostPartition(true, false, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadWriteAllWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.READ_WRITE_ALL; + + isPersistenceEnabled = true; + + checkLostPartition(true, false, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -180,6 +249,17 @@ public void testReadWriteSafeAfterKillTwoNodes() throws Exception { checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeAfterKillTwoNodesWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -189,6 +269,17 @@ public void testReadWriteSafeAfterKillTwoNodesWithDelay() throws Exception { checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 20)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeAfterKillTwoNodesWithDelayWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 20)); + } + /** * @throws Exception if failed. */ @@ -200,6 +291,21 @@ public void testReadWriteSafeWithBackupsAfterKillThreeNodes() throws Exception { checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2, 1), Arrays.asList(0, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeWithBackupsAfterKillThreeNodesWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10043"); + + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + backups = 1; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2, 1), Arrays.asList(0, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -209,6 +315,17 @@ public void testReadWriteSafeAfterKillCrd() throws Exception { checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeAfterKillCrdWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -220,6 +337,19 @@ public void testReadWriteSafeWithBackups() throws Exception { checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeWithBackupsWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + backups = 1; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -232,12 +362,81 @@ public void testReadWriteSafeWithBackupsAfterKillCrd() throws Exception { } /** - * @param topChanger topology changer. * @throws Exception if failed. */ - public void testIgnore(TopologyChanger topChanger) throws Exception { + public void testReadWriteSafeWithBackupsAfterKillCrdWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + backups = 1; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); + } + + /** + * @throws Exception if failed. + */ + public void testIgnore() throws Exception { fail("https://issues.apache.org/jira/browse/IGNITE-5078"); + partLossPlc = PartitionLossPolicy.IGNORE; + + checkIgnore(killSingleNode); + } + + /** + * @throws Exception if failed. + */ + public void testIgnoreWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5078"); + + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.IGNORE; + + isPersistenceEnabled = true; + + checkIgnore(killSingleNode); + } + + /** + * @throws Exception if failed. + */ + public void testIgnoreKillThreeNodes() throws Exception { + partLossPlc = PartitionLossPolicy.IGNORE; + + // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078. + // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed. + // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0); + TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Collections.singletonList(0), 0); + + checkIgnore(onlyCrdIsAlive); + } + + /** + * @throws Exception if failed. + */ + public void testIgnoreKillThreeNodesWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.IGNORE; + + isPersistenceEnabled = true; + + // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078. + // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed. + // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0); + TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Collections.singletonList(0), 0); + + checkIgnore(onlyCrdIsAlive); + } + + /** + * @param topChanger topology changer. + * @throws Exception if failed. + */ + private void checkIgnore(TopologyChanger topChanger) throws Exception { topChanger.changeTopology(); for (Ignite ig : G.allGrids()) { @@ -266,14 +465,14 @@ public void testIgnore(TopologyChanger topChanger) throws Exception { private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger topChanger) throws Exception { assert partLossPlc != null; - int part = topChanger.changeTopology().get(0); + List lostParts = topChanger.changeTopology(); // Wait for all grids (servers and client) have same topology version // to make sure that all nodes received map with lost partition. - GridTestUtils.waitForCondition(() -> { + boolean success = GridTestUtils.waitForCondition(() -> { AffinityTopologyVersion last = null; for (Ignite ig : G.allGrids()) { - AffinityTopologyVersion ver = ((IgniteEx) ig).context().cache().context().exchange().readyAffinityVersion(); + AffinityTopologyVersion ver = ((IgniteEx)ig).context().cache().context().exchange().readyAffinityVersion(); if (last != null && !last.equals(ver)) return false; @@ -284,35 +483,57 @@ private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger return true; }, 10000); + assertTrue("Failed to wait for new topology", success); + for (Ignite ig : G.allGrids()) { info("Checking node: " + ig.cluster().localNode().id()); IgniteCache cache = ig.cache(CACHE_NAME); - verifyCacheOps(canWrite, safe, part, ig); + verifyLostPartitions(ig, lostParts); - // Check we can read and write to lost partition in recovery mode. - IgniteCache recoverCache = cache.withPartitionRecover(); + verifyCacheOps(canWrite, safe, ig); - for (int lostPart : recoverCache.lostPartitions()) { - recoverCache.get(lostPart); - recoverCache.put(lostPart, lostPart); - } + validateQuery(safe, ig); - // Check that writing in recover mode does not clear partition state. - verifyCacheOps(canWrite, safe, part, ig); + // TODO withPartitionRecover doesn't work with BLT - https://issues.apache.org/jira/browse/IGNITE-10041. + if (!isPersistenceEnabled) { + // Check we can read and write to lost partition in recovery mode. + IgniteCache recoverCache = cache.withPartitionRecover(); + + for (int lostPart : recoverCache.lostPartitions()) { + recoverCache.get(lostPart); + recoverCache.put(lostPart, lostPart); + } - // Validate queries. - validateQuery(safe, part, ig); + // Check that writing in recover mode does not clear partition state. + verifyLostPartitions(ig, lostParts); + + verifyCacheOps(canWrite, safe, ig); + + validateQuery(safe, ig); + } } - // Check that partition state does not change after we start a new node. - IgniteEx grd = startGrid(3); + // Bring all nodes back. + for (int i : topChanger.killNodes) { + IgniteEx grd = startGrid(i); - info("Newly started node: " + grd.cluster().localNode().id()); + info("Newly started node: " + grd.cluster().localNode().id()); - for (Ignite ig : G.allGrids()) - verifyCacheOps(canWrite, safe, part, ig); + // Check that partition state does not change after we start each node. + // TODO With persistence enabled LOST partitions become OWNING after a node joins back - https://issues.apache.org/jira/browse/IGNITE-10044. + if (!isPersistenceEnabled) { + for (Ignite ig : G.allGrids()) { + verifyCacheOps(canWrite, safe, ig); + + // TODO Query effectively waits for rebalance due to https://issues.apache.org/jira/browse/IGNITE-10057 + // TODO and after resetLostPartition there is another OWNING copy in the cluster due to https://issues.apache.org/jira/browse/IGNITE-10058. + // TODO Uncomment after https://issues.apache.org/jira/browse/IGNITE-10058 is fixed. +// validateQuery(safe, ig); + } + } + } ignite(4).resetLostPartitions(Collections.singletonList(CACHE_NAME)); @@ -330,24 +551,40 @@ private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger cache.put(i, i); } + + for (int i = 0; i < parts; i++) { + checkQueryPasses(ig, false, i); + + if (shouldExecuteLocalQuery(ig, i)) + checkQueryPasses(ig, true, i); + + } + + checkQueryPasses(ig, false); } } /** - * + * @param node Node. + * @param lostParts Lost partition IDs. + */ + private void verifyLostPartitions(Ignite node, List lostParts) { + IgniteCache cache = node.cache(CACHE_NAME); + + Set actualSortedLostParts = new TreeSet<>(cache.lostPartitions()); + Set expSortedLostParts = new TreeSet<>(lostParts); + + assertEqualsCollections(expSortedLostParts, actualSortedLostParts); + } + + /** * @param canWrite {@code True} if writes are allowed. * @param safe {@code True} if lost partition should trigger exception. - * @param part Lost partition ID. * @param ig Ignite instance. */ - private void verifyCacheOps(boolean canWrite, boolean safe, int part, Ignite ig) { + private void verifyCacheOps(boolean canWrite, boolean safe, Ignite ig) { IgniteCache cache = ig.cache(CACHE_NAME); - Collection lost = cache.lostPartitions(); - - assertTrue("Failed to find expected lost partition [exp=" + part + ", lost=" + lost + ']', - lost.contains(part)); - int parts = ig.affinity(CACHE_NAME).partitions(); // Check read. @@ -395,7 +632,7 @@ private void verifyCacheOps(boolean canWrite, boolean safe, int part, Ignite ig) * @param nodes List of nodes to find partition. * @return List of partitions that aren't primary or backup for specified nodes. */ - protected List noPrimaryOrBackupPartition(List nodes) { + private List noPrimaryOrBackupPartition(List nodes) { Affinity aff = ignite(4).affinity(CACHE_NAME); List parts = new ArrayList<>(); @@ -424,15 +661,125 @@ protected List noPrimaryOrBackupPartition(List nodes) { * Validate query execution on a node. * * @param safe Safe flag. - * @param part Partition. * @param node Node. */ - protected void validateQuery(boolean safe, int part, Ignite node) { + private void validateQuery(boolean safe, Ignite node) { + // Get node lost and remaining partitions. + IgniteCache cache = node.cache(CACHE_NAME); + + Collection lostParts = cache.lostPartitions(); + + int part = cache.lostPartitions().stream().findFirst().orElseThrow(AssertionFailedError::new); + + Integer remainingPart = null; + + for (int i = 0; i < node.affinity(CACHE_NAME).partitions(); i++) { + if (lostParts.contains(i)) + continue; + + remainingPart = i; + + break; + } + + assertNotNull("Failed to find a partition that isn't lost", remainingPart); + + // 1. Check query against all partitions. + validateQuery0(safe, node); + + // 2. Check query against LOST partition. + validateQuery0(safe, node, part); + + // 3. Check query on remaining partition. + checkQueryPasses(node, false, remainingPart); + + if (shouldExecuteLocalQuery(node, remainingPart)) + checkQueryPasses(node, true, remainingPart); + + // 4. Check query over two partitions - normal and LOST. + validateQuery0(safe, node, part, remainingPart); + } + + /** + * Query validation routine. + * + * @param safe Safe flag. + * @param node Node. + * @param parts Partitions. + */ + private void validateQuery0(boolean safe, Ignite node, int... parts) { + if (safe) + checkQueryFails(node, false, parts); + else + checkQueryPasses(node, false, parts); + + if (shouldExecuteLocalQuery(node, parts)) { + if (safe) + checkQueryFails(node, true, parts); + else + checkQueryPasses(node, true, parts); + } + } + + /** + * @return true if the given node is primary for all given partitions. + */ + private boolean shouldExecuteLocalQuery(Ignite node, int... parts) { + if (parts == null || parts.length == 0) + return false; + + int numOfPrimaryParts = 0; + + for (int nodePrimaryPart : node.affinity(CACHE_NAME).primaryPartitions(node.cluster().localNode())) { + for (int part : parts) { + if (part == nodePrimaryPart) + numOfPrimaryParts++; + } + } + + return numOfPrimaryParts == parts.length; + } + + /** + * @param node Node. + * @param loc Local flag. + * @param parts Partitions. + */ + protected void checkQueryPasses(Ignite node, boolean loc, int... parts) { + // Scan queries don't support multiple partitions. + if (parts != null && parts.length > 1) + return; + + // TODO Local scan queries fail in non-safe modes - https://issues.apache.org/jira/browse/IGNITE-10059. + if (loc) + return; + + IgniteCache cache = node.cache(CACHE_NAME); + + ScanQuery qry = new ScanQuery(); + + if (parts != null && parts.length > 0) + qry.setPartition(parts[0]); + + if (loc) + qry.setLocal(true); + + cache.query(qry).getAll(); + } + + /** + * @param node Node. + * @param loc Local flag. + * @param parts Partitions. + */ + protected void checkQueryFails(Ignite node, boolean loc, int... parts) { + // TODO Scan queries never fail due to partition loss - https://issues.apache.org/jira/browse/IGNITE-9902. + // TODO Need to add an actual check after https://issues.apache.org/jira/browse/IGNITE-9902 is fixed. // No-op. } /** */ - class TopologyChanger { + private class TopologyChanger { /** Flag to delay partition exchange */ private boolean delayExchange; @@ -451,7 +798,7 @@ class TopologyChanger { * @param aliveNodes List of nodes to be alive. * @param stopDelay Delay between stopping nodes. */ - public TopologyChanger(boolean delayExchange, List killNodes, List aliveNodes, + private TopologyChanger(boolean delayExchange, List killNodes, List aliveNodes, long stopDelay) { this.delayExchange = delayExchange; this.killNodes = killNodes; @@ -463,9 +810,12 @@ public TopologyChanger(boolean delayExchange, List killNodes, List changeTopology() throws Exception { + private List changeTopology() throws Exception { startGrids(4); + if (isPersistenceEnabled) + grid(0).cluster().active(true); + Affinity aff = ignite(0).affinity(CACHE_NAME); for (int i = 0; i < aff.partitions(); i++) @@ -497,7 +847,6 @@ protected List changeTopology() throws Exception { lostMap.add(semaphoreMap); - grid(i).events().localListen(new P1() { @Override public boolean apply(Event evt) { assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; @@ -512,7 +861,6 @@ protected List changeTopology() throws Exception { return true; } }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST); - } if (delayExchange) @@ -549,5 +897,4 @@ protected List changeTopology() throws Exception { return parts; } } - } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index b30aa2fedd4c1..87c8ce938be5d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -47,6 +47,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.PartitionLossPolicy; import org.apache.ignite.cache.query.QueryCancelledException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.DiscoveryEvent; @@ -112,6 +113,8 @@ import static java.util.Collections.singletonList; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT; +import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE; +import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE; import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE; import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.checkActive; import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.mvccEnabled; @@ -1693,6 +1696,24 @@ private NodesForPartitionsResult nodesForPartitions(List cacheIds, Affi Map partsMap = null; Map qryMap = null; + for (int cacheId : cacheIds) { + GridCacheContext cctx = cacheContext(cacheId); + + PartitionLossPolicy plc = cctx.config().getPartitionLossPolicy(); + + if (plc != READ_ONLY_SAFE && plc != READ_WRITE_SAFE) + continue; + + Collection lostParts = cctx.topology().lostPartitions(); + + for (int part : lostParts) { + if (parts == null || Arrays.binarySearch(parts, part) >= 0) { + throw new CacheException("Failed to execute query because cache partition has been " + + "lost [cacheName=" + cctx.name() + ", part=" + part + ']'); + } + } + } + if (isPreloadingActive(cacheIds)) { if (isReplicatedOnly) nodes = replicatedUnstableDataNodes(cacheIds, qryId); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java index f2085994b0cd0..a31a1c6110d50 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java @@ -23,8 +23,6 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest; -import java.util.Collection; - /** * Partition loss policy test with enabled indexing. */ @@ -39,80 +37,27 @@ public class IndexingCachePartitionLossPolicySelfTest extends IgniteCachePartiti } /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override protected void validateQuery(boolean safe, int part, Ignite node) { - // Get node lost and remaining partitions. - IgniteCache cache = node.cache(CACHE_NAME); - - Collection lostParts = cache.lostPartitions(); - - Integer remainingPart = null; - - for (int i = 0; i < node.affinity(CACHE_NAME).partitions(); i++) { - if (lostParts.contains(i)) - continue; - - remainingPart = i; - - break; - } - - // Determine whether local query should be executed on that node. - boolean execLocQry = false; - - for (int nodePrimaryPart : node.affinity(CACHE_NAME).primaryPartitions(node.cluster().localNode())) { - if (part == nodePrimaryPart) { - execLocQry = true; - - break; - } - } - - // 1. Check query against all partitions. - validateQuery0(safe, node, false); - - // TODO: https://issues.apache.org/jira/browse/IGNITE-7039 -// if (execLocQry) -// validateQuery0(safe, node, true); - - // 2. Check query against LOST partition. - validateQuery0(safe, node, false, part); + protected void checkQueryPasses(Ignite node, boolean loc, int... parts) { + executeQuery(node, loc, parts); + } - // TODO: https://issues.apache.org/jira/browse/IGNITE-7039 -// if (execLocQry) -// validateQuery0(safe, node, true, part); + /** {@inheritDoc} */ + protected void checkQueryFails(Ignite node, boolean loc, int... parts) { + // TODO: Local queries ignore partition loss, see https://issues.apache.org/jira/browse/IGNITE-7039. + if (loc) + return; - // 3. Check query on remaining partition. - if (remainingPart != null) { - executeQuery(node, false, remainingPart); + try { + executeQuery(node, loc, parts); - // 4. Check query over two partitions - normal and LOST. - validateQuery0(safe, node, false, part, remainingPart); + fail("Exception is not thrown."); } - } - - /** - * Query validation routine. - * - * @param safe Safe flag. - * @param node Node. - * @param loc Local flag. - * @param parts Partitions. - */ - private void validateQuery0(boolean safe, Ignite node, boolean loc, int... parts) { - if (safe) { - try { - executeQuery(node, loc, parts); + catch (Exception e) { + boolean exp = e.getMessage() != null && + e.getMessage().contains("Failed to execute query because cache partition has been lost"); - fail("Exception is not thrown."); - } - catch (Exception e) { - assertTrue(e.getMessage(), e.getMessage() != null && - e.getMessage().contains("Failed to execute query because cache partition has been lost")); - } - } - else { - executeQuery(node, loc, parts); + if (!exp) + throw e; } } From af3c8f686953350619dba8d13c9296cbc0198d29 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Fri, 2 Nov 2018 14:52:31 +0300 Subject: [PATCH 068/403] IGNITE-10123 Disable events in IgniteLogicalRecoveryTest - Fixes #5237. Signed-off-by: Alexey Goncharuk --- .../persistence/db/IgniteLogicalRecoveryTest.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java index 584196f6797b2..425f9b955a31d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java @@ -61,6 +61,9 @@ * A set of tests that check correctness of logical recovery performed during node start. */ public class IgniteLogicalRecoveryTest extends GridCommonAbstractTest { + /** */ + private static final int[] EVTS_DISABLED = {}; + /** Shared group name. */ private static final String SHARED_GROUP_NAME = "group"; @@ -73,9 +76,12 @@ public class IgniteLogicalRecoveryTest extends GridCommonAbstractTest { /** Io factory. */ private FileIOFactory ioFactory; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + cfg.setIncludeEventTypes(EVTS_DISABLED); + cfg.setConsistentId(igniteInstanceName); cfg.setCacheConfiguration( @@ -381,7 +387,7 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { /** * */ - private class AggregateCacheLoader { + private static class AggregateCacheLoader { /** Ignite. */ IgniteEx ignite; @@ -432,7 +438,7 @@ public void stop() throws IgniteCheckedException { /** * @param ignite Ignite. */ - public void consistencyCheck(IgniteEx ignite) throws IgniteCheckedException { + public void consistencyCheck(IgniteEx ignite) { if (cacheLoaders != null) for (CacheLoader cacheLoader : cacheLoaders.keySet()) cacheLoader.consistencyCheck(ignite); @@ -567,9 +573,12 @@ static String arrayToString(int[] arr) { return "null"; StringBuilder sb = new StringBuilder(); + sb.append('['); + for (int i = 0; i < Math.min(arr.length, 10); i++) - sb.append(i + ","); + sb.append(i).append(","); + sb.append(']'); return sb.toString(); From aa6a55df0fd794a76d3cf7847812dc655b6d796e Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 2 Nov 2018 14:56:41 +0300 Subject: [PATCH 069/403] IGNITE-6856: SQL: Fixed query handling from thin client side. This closes #5238. --- .../processors/query/h2/IgniteH2Indexing.java | 30 +++++++++++++++++++ .../query/h2/sql/GridSqlQueryParser.java | 22 ++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index d5666e80d769c..820792f1d22a4 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.processors.cache.CacheEntryImpl; import org.apache.ignite.internal.processors.cache.CacheObjectUtils; import org.apache.ignite.internal.processors.cache.CacheObjectValueContext; +import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; @@ -178,6 +179,7 @@ import org.apache.ignite.marshaller.Marshaller; import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.security.SecurityPermission; import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.apache.ignite.spi.indexing.IndexingQueryFilterImpl; @@ -1123,6 +1125,14 @@ else if (DdlStatementsProcessor.isDdlStatement(p)) { throw new IgniteSQLException("SELECT FOR UPDATE query requires transactional " + "cache with MVCC enabled.", IgniteQueryErrorCode.UNSUPPORTED_OPERATION); + if (this.ctx.security().enabled()) { + GridSqlQueryParser parser = new GridSqlQueryParser(false); + + parser.parse(p); + + checkSecurity(parser.cacheIds()); + } + GridNearTxSelectForUpdateFuture sfuFut = null; int opTimeout = qryTimeout; @@ -2313,6 +2323,9 @@ private List>> doRunPrepared(String schemaNa checkQueryType(qry, true); + if (ctx.security().enabled()) + checkSecurity(twoStepQry.cacheIds()); + return Collections.singletonList(doRunDistributedQuery(schemaName, qry, twoStepQry, meta, keepBinary, startTx, tracker, cancel)); } @@ -2327,6 +2340,23 @@ private List>> doRunPrepared(String schemaNa } } + /** + * Check security access for caches. + * + * @param cacheIds Cache IDs. + */ + private void checkSecurity(Collection cacheIds) { + if (F.isEmpty(cacheIds)) + return; + + for (Integer cacheId : cacheIds) { + DynamicCacheDescriptor desc = ctx.cache().cacheDescriptor(cacheId); + + if (desc != null) + ctx.security().authorize(desc.cacheName(), SecurityPermission.CACHE_READ, null); + } + } + /** * Parse and split query if needed, cache either two-step query or statement. * @param schemaName Schema name. diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java index 856951fe21d01..ab661ce59b9c4 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java @@ -21,6 +21,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.IdentityHashMap; @@ -1763,6 +1764,27 @@ public GridCacheContext getFirstPartitionedCache() { return null; } + /** + * @return All known cache IDs. + */ + public Collection cacheIds() { + ArrayList res = new ArrayList<>(1); + + for (Object o : h2ObjToGridObj.values()) { + if (o instanceof GridSqlAlias) + o = GridSqlAlias.unwrap((GridSqlAst)o); + + if (o instanceof GridSqlTable) { + GridH2Table tbl = ((GridSqlTable)o).dataTable(); + + if (tbl != null) + res.add(tbl.cacheId()); + } + } + + return res; + } + /** * @param stmt Prepared statement. * @return Parsed AST. From fa192a9b1368179e6d20d03874a16f4365de93d3 Mon Sep 17 00:00:00 2001 From: ibessonov Date: Fri, 2 Nov 2018 12:45:02 +0300 Subject: [PATCH 070/403] IGNITE-9972 Fixed memory leak of deleted entries in partition map - Fixes #5196. Signed-off-by: Alexey Goncharuk (cherry picked from commit bbcf5b4) --- .../configuration/CacheConfiguration.java | 9 +- .../processors/cache/GridCacheMapEntry.java | 50 +++++- .../GridDistributedCacheEntry.java | 18 +- .../dht/topology/GridDhtLocalPartition.java | 10 +- .../cache/local/GridLocalCacheEntry.java | 22 ++- .../internal/worker/WorkersRegistry.java | 2 +- .../IgniteTxConcurrentRemoveObjectsTest.java | 165 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite9.java | 3 + 8 files changed, 245 insertions(+), 34 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java index 795fcfde9ce8e..05be893392b02 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java @@ -50,9 +50,7 @@ import org.apache.ignite.cache.store.CacheStore; import org.apache.ignite.cache.store.CacheStoreSessionListener; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.spi.encryption.EncryptionSpi; import org.apache.ignite.internal.binary.BinaryContext; -import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi; import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.A; @@ -60,6 +58,8 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.plugin.CachePluginConfiguration; +import org.apache.ignite.spi.encryption.EncryptionSpi; +import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi; import org.jetbrains.annotations.Nullable; /** @@ -318,7 +318,7 @@ public class CacheConfiguration extends MutableConfiguration { private long rebalanceThrottle = DFLT_REBALANCE_THROTTLE; /** */ - private CacheInterceptor interceptor; + private CacheInterceptor interceptor; /** */ private Class[] sqlFuncCls; @@ -1630,9 +1630,8 @@ public CacheConfiguration setMaxQueryIteratorsCount(int maxQryIterCnt) { * * @return Cache interceptor. */ - @SuppressWarnings({"unchecked"}) @Nullable public CacheInterceptor getInterceptor() { - return (CacheInterceptor)interceptor; + return interceptor; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 347e2607344a9..e09f3734c591f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -139,7 +139,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme /** * NOTE + *
* ==== + *
* Make sure to recalculate this value any time when adding or removing fields from entry. * The size should be count as follows: *
    @@ -147,9 +149,45 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme *
  • References: 8 each
  • *
  • Each nested object should be analyzed in the same way as above.
  • *
+ * ==== + *
+ *
    + *
  • Reference fields:
      + *
    • 8 : {@link #cctx}
    • + *
    • 8 : {@link #key}
    • + *
    • 8 : {@link #val}
    • + *
    • 8 : {@link #ver}
    • + *
    • 8 : {@link #extras}
    • + *
    • 8 : {@link #lock}
    • + *
    • 8 : {@link #listenerLock}
    • + *
    • 8 : {@link GridMetadataAwareAdapter#data}
    • + *
  • + *
  • Primitive fields:
      + *
    • 4 : {@link #hash}
    • + *
    • 1 : {@link #flags}
    • + *
  • + *
  • Extras:
      + *
    • 8 : {@link GridCacheEntryExtras#ttl()}
    • + *
    • 8 : {@link GridCacheEntryExtras#expireTime()}
    • + *
  • + *
  • Version:
      + *
    • 4 : {@link GridCacheVersion#topVer}
    • + *
    • 4 : {@link GridCacheVersion#nodeOrderDrId}
    • + *
    • 8 : {@link GridCacheVersion#order}
    • + *
  • + *
  • Key:
      + *
    • 8 : {@link CacheObjectAdapter#val}
    • + *
    • 8 : {@link CacheObjectAdapter#valBytes}
    • + *
    • 4 : {@link KeyCacheObjectImpl#part}
    • + *
  • + *
  • Value:
      + *
    • 8 : {@link CacheObjectAdapter#val}
    • + *
    • 8 : {@link CacheObjectAdapter#valBytes}
    • + *
  • + *
*/ - // 7 * 8 /*references*/ + 2 * 8 /*long*/ + 1 * 4 /*int*/ + 1 * 1 /*byte*/ + array at parent = 85 - private static final int SIZE_OVERHEAD = 85 /*entry*/ + 32 /* version */ + 4 * 7 /* key + val */; + private static final int SIZE_OVERHEAD = 8 * 8 /* references */ + 5 /* primitives */ + 16 /* extras */ + + 16 /* version */ + 20 /* key */ + 16 /* value */; /** Static logger to avoid re-creation. Made static for test purpose. */ protected static final AtomicReference logRef = new AtomicReference<>(); @@ -1828,9 +1866,6 @@ else if (log.isDebugEnabled()) unlockListenerReadLock(); } - if (deferred) - cctx.onDeferredDelete(this, newVer); - if (marked) { assert !deferred; @@ -4892,6 +4927,11 @@ private int extrasSize() { return extras != null ? extras.size() : 0; } + /** {@inheritDoc} */ + @Override public void txUnlock(IgniteInternalTx tx) throws GridCacheEntryRemovedException { + removeLock(tx.xidVersion()); + } + /** {@inheritDoc} */ @Override public void onUnlock() { // No-op. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java index ff636c7c9a9a2..b07451a6a2091 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java @@ -368,6 +368,8 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc GridCacheMvccCandidate doomed; + GridCacheVersion deferredDelVer; + CacheObject val; lockEntry(); @@ -406,11 +408,22 @@ public void removeExplicitNodeLocks(UUID nodeId) throws GridCacheEntryRemovedExc } val = this.val; + + deferredDelVer = this.ver; } finally { unlockEntry(); } + if (val == null) { + boolean deferred = cctx.deferredDelete() && !detached() && !isInternal(); + + if (deferred) { + if (deferredDelVer != null) + cctx.onDeferredDelete(this, deferredDelVer); + } + } + if (log.isDebugEnabled()) log.debug("Removed lock candidate from entry [doomed=" + doomed + ", owner=" + owner + ", prev=" + prev + ", entry=" + this + ']'); @@ -705,11 +718,6 @@ public void recheck() { } } - /** {@inheritDoc} */ - @Override public final void txUnlock(IgniteInternalTx tx) throws GridCacheEntryRemovedException { - removeLock(tx.xidVersion()); - } - /** * @param emptyBefore Empty flag before operation. * @param emptyAfter Empty flag after operation. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 2ddc0d447ceba..588c8b4068f18 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -79,15 +79,7 @@ */ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements Comparable, GridReservable { /** */ - private static final GridCacheMapEntryFactory ENTRY_FACTORY = new GridCacheMapEntryFactory() { - @Override public GridCacheMapEntry create( - GridCacheContext ctx, - AffinityTopologyVersion topVer, - KeyCacheObject key - ) { - return new GridDhtCacheEntry(ctx, topVer, key); - } - }; + private static final GridCacheMapEntryFactory ENTRY_FACTORY = GridDhtCacheEntry::new; /** Maximum size for delete queue. */ public static final int MAX_DELETE_QUEUE_SIZE = Integer.getInteger(IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE, 200_000); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java index 412d4d30e4e76..e26174a2adef5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java @@ -257,15 +257,6 @@ public void recheck() { } } - /** - * Unlocks lock if it is currently owned. - * - * @param tx Transaction to unlock. - */ - @Override public void txUnlock(IgniteInternalTx tx) throws GridCacheEntryRemovedException { - removeLock(tx.xidVersion()); - } - /** * Releases local lock. */ @@ -327,6 +318,8 @@ private void releaseLocal(long threadId) { GridCacheMvccCandidate doomed; + GridCacheVersion deferredDelVer; + lockEntry(); try { @@ -351,11 +344,22 @@ private void releaseLocal(long threadId) { } val = this.val; + + deferredDelVer = this.ver; } finally { unlockEntry(); } + if (val == null) { + boolean deferred = cctx.deferredDelete() && !detached() && !isInternal(); + + if (deferred) { + if (deferredDelVer != null) + cctx.onDeferredDelete(this, deferredDelVer); + } + } + if (doomed != null) checkThreadChain(doomed); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java index 153b289951f0d..3d45f3fc5c404 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java @@ -212,7 +212,7 @@ public void setSystemWorkerBlockedTimeout(long val) { // That is, if worker is dead, but still resides in registeredWorkers // then something went wrong, the only extra thing is to test // whether the iterator refers to actual state of registeredWorkers. - GridWorker worker0 = registeredWorkers.get(worker.runner().getName()); + GridWorker worker0 = registeredWorkers.get(runner.getName()); if (worker0 != null && worker0 == worker) workerFailedHnd.apply(worker, SYSTEM_WORKER_TERMINATION); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java new file mode 100644 index 0000000000000..a5b2cb71a4ef9 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.UUID; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; + +import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL; +import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; + +/** + * + */ +public class IgniteTxConcurrentRemoveObjectsTest extends GridCommonAbstractTest { + /** Cache partitions. */ + private static final int CACHE_PARTITIONS = 16; + + /** Cache entries count. */ + private static final int CACHE_ENTRIES_COUNT = 512 * CACHE_PARTITIONS; + + /** IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** New value for {@link IgniteSystemProperties#IGNITE_CACHE_REMOVED_ENTRIES_TTL} property. */ + private static final long newIgniteCacheRemovedEntriesTtl = 50L; + + /** Old value of {@link IgniteSystemProperties#IGNITE_CACHE_REMOVED_ENTRIES_TTL} property. */ + private static long oldIgniteCacheRmvEntriesTtl; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + oldIgniteCacheRmvEntriesTtl = Long.getLong(IGNITE_CACHE_REMOVED_ENTRIES_TTL, 10_000); + + System.setProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL, Long.toString(newIgniteCacheRemovedEntriesTtl)); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + System.setProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL, Long.toString(oldIgniteCacheRmvEntriesTtl)); + + super.afterTestsStopped(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + return cfg; + } + + /** + * @return Cache configuration. + */ + private CacheConfiguration getCacheCfg() { + CacheConfiguration ccfg = new CacheConfiguration<>(); + + ccfg.setName(DEFAULT_CACHE_NAME); + + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + ccfg.setAffinity(new RendezvousAffinityFunction().setPartitions(CACHE_PARTITIONS)); + + return ccfg; + } + + /** + * Too many deletes in single transaction may overflow {@link GridDhtLocalPartition#rmvQueue} and entries will be + * deleted synchronously in {@link GridDhtLocalPartition#onDeferredDelete(int, KeyCacheObject, GridCacheVersion)}. + * This should not corrupt internal map state in {@link GridDhtLocalPartition}. + * + * @throws Exception If failed. + */ + public void testTxLeavesObjectsInLocalPartition() throws Exception { + IgniteEx igniteEx = startGrid(getConfiguration()); + + igniteEx.getOrCreateCache(getCacheCfg()); + + try (IgniteDataStreamer dataStreamer = igniteEx.dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < CACHE_ENTRIES_COUNT; i++) + dataStreamer.addData(i, UUID.randomUUID().toString()); + } + + IgniteEx client = startGrid( + getConfiguration() + .setClientMode(true) + .setIgniteInstanceName(UUID.randomUUID().toString()) + ); + + awaitPartitionMapExchange(); + + assertEquals(CACHE_ENTRIES_COUNT, client.getOrCreateCache(DEFAULT_CACHE_NAME).size()); + + try (Transaction tx = client.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) { + IgniteCache cache = client.getOrCreateCache(getCacheCfg()); + + for (int v = 0; v < CACHE_ENTRIES_COUNT; v++) { + cache.get(v); + + cache.remove(v); + } + + tx.commit(); + } + + GridTestUtils.waitForCondition( + () -> igniteEx.context().cache().cacheGroups().stream() + .filter(CacheGroupContext::userCache) + .flatMap(cgctx -> cgctx.topology().localPartitions().stream()) + .mapToInt(GridDhtLocalPartition::internalSize) + .max().orElse(-1) == 0, + newIgniteCacheRemovedEntriesTtl * 10 + ); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index 7dba46114ccbc..4b47b417218d2 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; import org.apache.ignite.internal.processors.cache.transactions.TxDataConsistencyOnCommitFailureTest; import org.apache.ignite.testframework.junits.GridAbstractTest; @@ -50,6 +51,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class); + suite.addTestSuite(IgniteTxConcurrentRemoveObjectsTest.class); + suite.addTestSuite(TxDataConsistencyOnCommitFailureTest.class); return suite; From 8cf79fcebfe3e33b23589cb1e2368fdc5ae89d6f Mon Sep 17 00:00:00 2001 From: Alexey Platonov Date: Fri, 5 Oct 2018 11:57:07 +0300 Subject: [PATCH 071/403] IGNITE-9675 Fixed deadlock on Ignite#active() and concurrent node stop - Fixes #4822. Signed-off-by: Alexey Goncharuk (cherry picked from commit 754c7337de123ac44e2816d2a55ab6f76cd03eac) --- .../cluster/GridClusterStateProcessor.java | 75 +++++++++++-------- .../cluster/IGridClusterStateProcessor.java | 13 +++- .../processors/task/GridTaskProcessor.java | 52 +++++++++---- .../processors/igfs/IgfsIgniteMock.java | 12 +-- 4 files changed, 97 insertions(+), 55 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index d4928469d088c..e0a4f0a4528fd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -17,18 +17,6 @@ package org.apache.ignite.internal.processors.cluster; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteCompute; @@ -59,8 +47,11 @@ import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.C1; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.CI2; import org.apache.ignite.internal.util.typedef.F; @@ -79,6 +70,19 @@ import org.apache.ignite.spi.discovery.DiscoveryDataBag; import org.jetbrains.annotations.Nullable; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; + import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.STATE_PROC; @@ -173,6 +177,11 @@ public boolean compatibilityMode() { /** {@inheritDoc} */ @Override public boolean publicApiActiveState(boolean waitForTransition) { + return publicApiActiveStateAsync(waitForTransition).get(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture publicApiActiveStateAsync(boolean asyncWaitForTransition) { if (ctx.isDaemon()) return sendComputeCheckGlobalState(); @@ -184,32 +193,34 @@ public boolean compatibilityMode() { Boolean transitionRes = globalState.transitionResult(); if (transitionRes != null) - return transitionRes; + return new IgniteFinishedFutureImpl<>(transitionRes); else { - if (waitForTransition) { - GridFutureAdapter fut = transitionFuts.get(globalState.transitionRequestId()); + GridFutureAdapter fut = transitionFuts.get(globalState.transitionRequestId()); + if (fut != null) { + if (asyncWaitForTransition) { + return new IgniteFutureImpl<>(fut.chain(new C1, Boolean>() { + @Override public Boolean apply(IgniteInternalFuture fut) { + Boolean res = globalState.transitionResult(); - if (fut != null) { - try { - fut.get(); - } - catch (IgniteCheckedException ex) { - throw new IgniteException(ex); - } + assert res != null; + + return res; + } + })); } + else + return new IgniteFinishedFutureImpl<>(globalState.baselineChanged()); + } - transitionRes = globalState.transitionResult(); + transitionRes = globalState.transitionResult(); - assert transitionRes != null; + assert transitionRes != null; - return transitionRes; - } - else - return globalState.baselineChanged(); + return new IgniteFinishedFutureImpl<>(transitionRes); } } else - return globalState.active(); + return new IgniteFinishedFutureImpl<>(globalState.active()); } /** {@inheritDoc} */ @@ -1066,7 +1077,7 @@ private void sendComputeChangeGlobalState( * * @return Cluster state, {@code True} if cluster active, {@code False} if inactive. */ - private boolean sendComputeCheckGlobalState() { + private IgniteFuture sendComputeCheckGlobalState() { AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx(); if (log.isInfoEnabled()) { @@ -1079,11 +1090,11 @@ private boolean sendComputeCheckGlobalState() { ClusterGroupAdapter clusterGroupAdapter = (ClusterGroupAdapter)ctx.cluster().get().forServers(); if (F.isEmpty(clusterGroupAdapter.nodes())) - return false; + return new IgniteFinishedFutureImpl<>(false); IgniteCompute comp = clusterGroupAdapter.compute(); - return comp.call(new IgniteCallable() { + return comp.callAsync(new IgniteCallable() { @IgniteInstanceResource private Ignite ig; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java index bc72a516b860a..1f677ed0cdef3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java @@ -18,9 +18,6 @@ package org.apache.ignite.internal.processors.cluster; -import java.util.Collection; -import java.util.Map; -import java.util.UUID; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.BaselineNode; import org.apache.ignite.cluster.ClusterNode; @@ -29,8 +26,13 @@ import org.apache.ignite.internal.processors.GridProcessor; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.StateChangeRequest; +import org.apache.ignite.lang.IgniteFuture; import org.jetbrains.annotations.Nullable; +import java.util.Collection; +import java.util.Map; +import java.util.UUID; + /** * */ @@ -40,6 +42,11 @@ public interface IGridClusterStateProcessor extends GridProcessor { */ boolean publicApiActiveState(boolean waitForTransition); + /** + * @return Cluster state to be used on public API. + */ + IgniteFuture publicApiActiveStateAsync(boolean waitForTransition); + /** * @param discoCache Discovery data cache. * @return If transition is in progress returns future which is completed when transition finishes. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index f4d3ccfd0ae2a..f663d8fca8876 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -17,18 +17,6 @@ package org.apache.ignite.internal.processors.task; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.LongAdder; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; @@ -54,6 +42,7 @@ import org.apache.ignite.internal.GridTaskSessionRequest; import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException; import org.apache.ignite.internal.IgniteDeploymentCheckedException; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException; import org.apache.ignite.internal.managers.communication.GridIoManager; @@ -69,6 +58,7 @@ import org.apache.ignite.internal.util.typedef.C1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.lang.IgniteFuture; @@ -78,10 +68,24 @@ import org.apache.ignite.plugin.security.SecurityPermission; import org.jetbrains.annotations.Nullable; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; + +import static org.apache.ignite.events.EventType.EVT_MANAGEMENT_TASK_STARTED; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.events.EventType.EVT_TASK_SESSION_ATTR_SET; -import static org.apache.ignite.events.EventType.EVT_MANAGEMENT_TASK_STARTED; import static org.apache.ignite.internal.GridTopic.TOPIC_JOB_SIBLINGS; import static org.apache.ignite.internal.GridTopic.TOPIC_TASK; import static org.apache.ignite.internal.GridTopic.TOPIC_TASK_CANCEL; @@ -189,7 +193,24 @@ private IgniteClientDisconnectedCheckedException disconnectedError(@Nullable Ign /** {@inheritDoc} */ @SuppressWarnings("TooBroadScope") @Override public void onKernalStop(boolean cancel) { - lock.writeLock(); + boolean interrupted = false; + + while (true) { + try { + if (lock.tryWriteLock(1, TimeUnit.SECONDS)) + break; + else { + LT.warn(log, "Still waiting to acquire write lock on stop"); + + U.sleep(50); + } + } + catch (IgniteInterruptedCheckedException | InterruptedException e) { + LT.warn(log, "Stopping thread was interrupted while waiting for write lock (will wait anyway)"); + + interrupted = true; + } + } try { stopping = true; @@ -198,6 +219,9 @@ private IgniteClientDisconnectedCheckedException disconnectedError(@Nullable Ign } finally { lock.writeUnlock(); + + if (interrupted) + Thread.currentThread().interrupt(); } startLatch.countDown(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java index a0ce28559e847..8159db7773537 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java @@ -19,6 +19,7 @@ import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.DataRegionMetricsAdapter; +import org.apache.ignite.DataStorageMetrics; import org.apache.ignite.DataStorageMetricsAdapter; import org.apache.ignite.IgniteAtomicLong; import org.apache.ignite.IgniteAtomicReference; @@ -41,7 +42,6 @@ import org.apache.ignite.IgniteServices; import org.apache.ignite.IgniteSet; import org.apache.ignite.IgniteTransactions; -import org.apache.ignite.DataStorageMetrics; import org.apache.ignite.MemoryMetrics; import org.apache.ignite.PersistenceMetrics; import org.apache.ignite.cache.affinity.Affinity; @@ -66,10 +66,10 @@ import org.apache.ignite.plugin.PluginNotFoundException; import org.jetbrains.annotations.Nullable; +import javax.cache.CacheException; import java.util.Collection; import java.util.Collections; import java.util.concurrent.ExecutorService; -import javax.cache.CacheException; /** * Mocked Ignite implementation for IGFS tests. @@ -182,13 +182,13 @@ public IgfsIgniteMock(@Nullable String name, IgniteFileSystem igfs) { return null; } - @Override - public boolean isRebalanceEnabled() { + /** {@inheritDoc} */ + @Override public boolean isRebalanceEnabled() { return true; } - @Override - public void rebalanceEnabled(boolean rebalanceEnabled) { + /** {@inheritDoc} */ + @Override public void rebalanceEnabled(boolean rebalanceEnabled) { throwUnsupported(); } From 7dc179ba55a9c884341d5bc3c79526791d0a5acc Mon Sep 17 00:00:00 2001 From: vd-pyatkov Date: Thu, 8 Nov 2018 15:12:11 +0300 Subject: [PATCH 072/403] IGNITE-9890 Avoid sorting partitions when possible during construction of CachePartitionPartialCountersMap - Fixes #5060. Signed-off-by: Alexey Goncharuk --- .../GridCachePartitionExchangeManager.java | 8 ++++- .../CachePartitionPartialCountersMap.java | 2 +- .../GridDhtPartitionsSingleMessage.java | 29 ------------------- .../GridDhtPartitionsStateValidator.java | 4 +-- 4 files changed, 10 insertions(+), 33 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index b0e0d0d31aa49..e3d9c65c7d625 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -134,6 +134,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE; import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; +import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.nextDumpTimeout; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.DFLT_PRELOAD_RESEND_TIMEOUT; @@ -1330,7 +1331,12 @@ private void sendLocalPartitions( @NotNull Collection grps ) { GridDhtPartitionsSingleMessage m = - createPartitionsSingleMessage(id, cctx.kernalContext().clientNode(), false, false, null, grps); + createPartitionsSingleMessage(id, + cctx.kernalContext().clientNode(), + false, + node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0, + null, + grps); if (log.isTraceEnabled()) log.trace("Sending local partitions [nodeId=" + node.id() + ", msg=" + m + ']'); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java index 9fc7f941a2188..2ff875d5e1877 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java @@ -34,7 +34,7 @@ public class CachePartitionPartialCountersMap implements Serializable { private static final long serialVersionUID = 0L; /** */ - static final IgniteProductVersion PARTIAL_COUNTERS_MAP_SINCE = IgniteProductVersion.fromString("2.1.4"); + public static final IgniteProductVersion PARTIAL_COUNTERS_MAP_SINCE = IgniteProductVersion.fromString("2.1.4"); /** */ public static final CachePartitionPartialCountersMap EMPTY = new CachePartitionPartialCountersMap(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index 7dd34f80e00bc..088fb31d7f715 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -245,35 +245,6 @@ public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int p return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt); } - /** - * @param grpId Cache group ID. - * @param partsCnt Total cache partitions. - * @return Partition update counters. - */ - @SuppressWarnings("unchecked") - public CachePartitionPartialCountersMap partitionUpdateCountersUnsorted(int grpId, int partsCnt) { - Object res = partCntrs == null ? null : partCntrs.get(grpId); - - if (res == null) - return CachePartitionPartialCountersMap.EMPTY; - - if (res instanceof CachePartitionPartialCountersMap) - return (CachePartitionPartialCountersMap)res; - - assert res instanceof Map : res; - - Map> map = (Map>)res; - - CachePartitionPartialCountersMap partCounersMap = new CachePartitionPartialCountersMap(partsCnt); - - for (Map.Entry> e : map.entrySet()) - partCounersMap.add(e.getKey(), e.getValue().get1(), e.getValue().get2()); - - partCounersMap.trim(); - - return partCounersMap; - } - /** * Adds partition sizes map for specified {@code grpId} to the current message. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java index 63fe926acc60f..4a0e21832764f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java @@ -190,7 +190,7 @@ public Map> validatePartitionsUpdateCounters( final GridDhtPartitionsSingleMessage message = e.getValue(); - CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions); + CachePartitionPartialCountersMap countersMap = message.partitionUpdateCounters(top.groupId(), partitions); Map sizesMap = message.partitionSizes(top.groupId()); @@ -250,7 +250,7 @@ public Map> validatePartitionsSizes( final GridDhtPartitionsSingleMessage message = e.getValue(); - CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions); + CachePartitionPartialCountersMap countersMap = message.partitionUpdateCounters(top.groupId(), partitions); Map sizesMap = message.partitionSizes(top.groupId()); From 6694b5ddc29d6b7e8c9f8c1b06a638332b481957 Mon Sep 17 00:00:00 2001 From: zstan Date: Fri, 9 Nov 2018 11:26:14 +0300 Subject: [PATCH 073/403] IGNITE-10128 Race in read\write cache configurations. - Fixes #5241. Signed-off-by: Alexey Goncharuk --- .../file/FilePageStoreManager.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 0f01fd452268f..4a14c6b261018 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -186,6 +185,22 @@ public FilePageStoreManager(GridKernalContext ctx) { "DataStorageConfiguration#storagePath properties). " + "Current persistence store directory is: [" + tmpDir + "]"); } + + File[] files = storeWorkDir.listFiles(); + + for (File file : files) { + if (file.isDirectory()) { + File[] tmpFiles = file.listFiles((k, v) -> v.endsWith(CACHE_DATA_TMP_FILENAME)); + + if (tmpFiles != null) { + for (File tmpFile : tmpFiles) { + if (!tmpFile.delete()) + log.warning("Failed to delete temporary cache config file" + + "(make sure Ignite process has enough rights):" + file.getName()); + } + } + } + } } /** {@inheritDoc} */ @@ -376,17 +391,17 @@ public FilePageStoreManager(GridKernalContext ctx) { if (overwrite || !Files.exists(filePath) || Files.size(filePath) == 0) { File tmp = new File(file.getParent(), file.getName() + TMP_SUFFIX); - tmp.createNewFile(); + if (tmp.exists() && !tmp.delete()) { + log.warning("Failed to delete temporary cache config file" + + "(make sure Ignite process has enough rights):" + file.getName()); + } // Pre-existing file will be truncated upon stream open. try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(tmp))) { marshaller.marshal(cacheData, stream); } - if (file.exists()) - file.delete(); - - Files.move(tmp.toPath(), file.toPath()); + Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); } } catch (IOException ex) { @@ -787,20 +802,6 @@ else if (lockF.exists()) { for (File file : files) { if (file.isDirectory()) { - File[] tmpFiles = file.listFiles(new FilenameFilter() { - @Override public boolean accept(File dir, String name) { - return name.endsWith(CACHE_DATA_TMP_FILENAME); - } - }); - - if (tmpFiles != null) { - for (File tmpFile: tmpFiles) { - if (!tmpFile.delete()) - log.warning("Failed to delete temporary cache config file" + - "(make sure Ignite process has enough rights):" + file.getName()); - } - } - if (file.getName().startsWith(CACHE_DIR_PREFIX)) { File conf = new File(file, CACHE_DATA_FILENAME); From ae2377a796f67f92e44e935f358b2aafc34eac5d Mon Sep 17 00:00:00 2001 From: Fedotov Date: Fri, 9 Nov 2018 12:26:15 +0300 Subject: [PATCH 074/403] IGNITE-8619 Fixed remote node start ssh connection - Fixes #4314. Signed-off-by: Alexey Goncharuk --- .../util/nodestart/StartNodeCallableImpl.java | 107 ++++++++-- ...iteProjectionStartStopRestartSelfTest.java | 198 ++++-------------- 2 files changed, 127 insertions(+), 178 deletions(-) diff --git a/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java b/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java index 6014510002159..2e9905413cedb 100644 --- a/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java +++ b/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java @@ -31,6 +31,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.UUID; +import java.util.regex.Pattern; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterStartNodeResult; @@ -90,6 +91,9 @@ public class StartNodeCallableImpl implements StartNodeCallable { /** Connection timeout. */ private final int timeout; + /** Timeout processor. */ + private GridTimeoutProcessor proc; + /** Logger. */ @LoggerResource private transient IgniteLogger log; @@ -130,6 +134,8 @@ public StartNodeCallableImpl(IgniteRemoteStartSpecification spec, int timeout) { Session ses = null; try { + proc = ((IgniteEx)ignite).context().timeout(); + if (spec.key() != null) ssh.addIdentity(spec.key().getAbsolutePath()); @@ -274,7 +280,8 @@ public StartNodeCallableImpl(IgniteRemoteStartSpecification spec, int timeout) { info("Starting remote node with SSH command: " + startNodeCmd, spec.logger(), log); - shell(ses, startNodeCmd); + // Execute command via ssh and wait until id of new process will be found in the output. + shell(ses, startNodeCmd, "\\[(\\d)\\] (\\d)+"); findSuccess = "grep \"" + SUCCESSFUL_START_MSG + "\" " + scriptOutputPath; } @@ -315,8 +322,25 @@ public StartNodeCallableImpl(IgniteRemoteStartSpecification spec, int timeout) { * @throws IgniteInterruptedCheckedException If thread was interrupted while waiting. */ private void shell(Session ses, String cmd) throws JSchException, IOException, IgniteInterruptedCheckedException { + shell(ses, cmd, null); + } + + /** + * Executes command using {@code shell} channel. + * + * @param ses SSH session. + * @param cmd Command. + * @param regexp Regular expression to wait until it will be found in stream from node. + * @throws JSchException In case of SSH error. + * @throws IOException If IO error occurs. + * @throws IgniteInterruptedCheckedException If thread was interrupted while waiting. + */ + private void shell(Session ses, String cmd, String regexp) + throws JSchException, IOException, IgniteInterruptedCheckedException { ChannelShell ch = null; + GridTimeoutObject to = null; + try { ch = (ChannelShell)ses.openChannel("shell"); @@ -324,9 +348,44 @@ private void shell(Session ses, String cmd) throws JSchException, IOException, I try (PrintStream out = new PrintStream(ch.getOutputStream(), true)) { out.println(cmd); + } + + if (regexp != null) { + Pattern ptrn = Pattern.compile(regexp); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(ch.getInputStream()))) { + String line; + + boolean first = true; + + while ((line = reader.readLine()) != null) { + if (ptrn.matcher(line).find()) { + // Wait for a while until process from regexp really will be started. + U.sleep(50); + + break; + } + else if (first) { + to = initTimer(cmd); + + first = false; + } + } + } + catch (InterruptedIOException ignore) { + // No-op. + } + finally { + if (to != null) { + boolean r = proc.removeTimeoutObject(to); + + assert r || to.endTime() <= U.currentTimeMillis() : "Timeout object was not removed: " + to; + } + } - U.sleep(EXECUTE_WAIT_TIME); } + else + U.sleep(EXECUTE_WAIT_TIME); } finally { if (ch != null && ch.isConnected()) @@ -425,10 +484,6 @@ private String exec(Session ses, final String cmd, String encoding) throws JSchE if (encoding == null) encoding = Charset.defaultCharset().name(); - IgniteEx grid = (IgniteEx)ignite; - - GridTimeoutProcessor proc = grid.context().timeout(); - GridTimeoutObject to = null; SB out = null; @@ -447,20 +502,7 @@ private String exec(Session ses, final String cmd, String encoding) throws JSchE out.a(line); if (first) { - to = new GridTimeoutObjectAdapter(EXECUTE_WAIT_TIME) { - /** */ - private final Thread thread = Thread.currentThread(); - - @Override public void onTimeout() { - thread.interrupt(); - } - - @Override public String toString() { - return S.toString("GridTimeoutObject", "cmd", cmd, "thread", thread); - } - }; - - assert proc.addTimeoutObject(to) : "Timeout object was not added: " + to; + to = initTimer(cmd); first = false; } @@ -485,6 +527,31 @@ private String exec(Session ses, final String cmd, String encoding) throws JSchE } } + /** + * Initialize timer to wait for command execution. + * + * @param cmd Command to log. + */ + private GridTimeoutObject initTimer(String cmd){ + GridTimeoutObject to = new GridTimeoutObjectAdapter(EXECUTE_WAIT_TIME) { + private final Thread thread = Thread.currentThread(); + + @Override public void onTimeout() { + thread.interrupt(); + } + + @Override public String toString() { + return S.toString("GridTimeoutObject", "cmd", cmd, "thread", thread); + } + }; + + boolean wasAdded = proc.addTimeoutObject(to); + + assert wasAdded : "Timeout object was not added: " + to; + + return to; + } + /** * Builds ignite.sh attributes to set up SSH username and password and log directory for started node. * diff --git a/modules/ssh/src/test/java/org/apache/ignite/internal/IgniteProjectionStartStopRestartSelfTest.java b/modules/ssh/src/test/java/org/apache/ignite/internal/IgniteProjectionStartStopRestartSelfTest.java index a91488caa985d..9ede98e1effcb 100644 --- a/modules/ssh/src/test/java/org/apache/ignite/internal/IgniteProjectionStartStopRestartSelfTest.java +++ b/modules/ssh/src/test/java/org/apache/ignite/internal/IgniteProjectionStartStopRestartSelfTest.java @@ -101,7 +101,7 @@ public class IgniteProjectionStartStopRestartSelfTest extends GridCommonAbstract private static final String CUSTOM_CFG_ATTR_VAL = "true"; /** */ - private static final long WAIT_TIMEOUT = 40 * 1000; + private static final long WAIT_TIMEOUT = 90 * 1000; /** */ private String pwd; @@ -265,28 +265,14 @@ public void testStartThreeNodes() throws Exception { public void testStartThreeNodesAndDoEmptyCall() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); + startCheckNodes(); assert joinedCnt.get() == 3; assert leftCnt.get() == 0; assert ignite.cluster().nodes().size() == 3; - res = startNodes(ignite.cluster(), + Collection res = startNodes(ignite.cluster(), maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), false, 0, 16); @@ -304,28 +290,12 @@ public void testStartThreeNodesAndDoEmptyCall() throws Exception { public void testStartThreeNodesAndTryToStartOneNode() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); + startCheckNodes(); assert joinedCnt.get() == 3; assert leftCnt.get() == 0; - assert ignite.cluster().nodes().size() == 3; - - res = startNodes(ignite.cluster(), + Collection res = startNodes(ignite.cluster(), maps(Collections.singleton(HOST), pwd, key, 1, U.getIgniteHome(), CFG_NO_ATTR, null), false, 0, 16); @@ -343,30 +313,14 @@ public void testStartThreeNodesAndTryToStartOneNode() throws Exception { public void testStartFiveNodesInTwoCalls() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); + startCheckNodes(); assert joinedCnt.get() == 3; assert leftCnt.get() == 0; - assert ignite.cluster().nodes().size() == 3; - joinedLatch = new CountDownLatch(2); - res = startNodes(ignite.cluster(), + Collection res = startNodes(ignite.cluster(), maps(Collections.singleton(HOST), pwd, key, 5, U.getIgniteHome(), CFG_NO_ATTR, null), false, 0, 16); @@ -422,31 +376,15 @@ public void testStartFiveWithTwoSpecs() throws Exception { public void testStartThreeNodesAndRestart() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); + startCheckNodes(); assert joinedCnt.get() == 3; assert leftCnt.get() == 0; - assert ignite.cluster().nodes().size() == 3; - joinedLatch = new CountDownLatch(3); leftLatch = new CountDownLatch(3); - res = startNodes(ignite.cluster(), + Collection res = startNodes(ignite.cluster(), maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), true, 0, 16); @@ -511,7 +449,7 @@ public void testStopNodes() throws Exception { Collection res = startNodes(ignite.cluster(), maps(Collections.singleton(HOST), pwd, null, 3, U.getIgniteHome(), CFG_NO_ATTR, - null), false, 0, 16); + null), false, 0, 16); assert res.size() == 3; @@ -596,23 +534,7 @@ public void testStopNodesFiltered() throws Exception { public void testStopNodeById() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); - - assert ignite.cluster().nodes().size() == 3; + startCheckNodes(); leftLatch = new CountDownLatch(1); @@ -629,23 +551,7 @@ public void testStopNodeById() throws Exception { public void testStopNodesByIds() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); - - assert ignite.cluster().nodes().size() == 3; + startCheckNodes(); leftLatch = new CountDownLatch(2); @@ -669,23 +575,7 @@ public void testStopNodesByIds() throws Exception { public void testRestartNodes() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); - - assert ignite.cluster().nodes().size() == 3; + startCheckNodes(); joinedLatch = new CountDownLatch(3); leftLatch = new CountDownLatch(3); @@ -762,23 +652,7 @@ public void testRestartNodesFiltered() throws Exception { public void testRestartNodeById() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); - - assert ignite.cluster().nodes().size() == 3; + startCheckNodes(); joinedLatch = new CountDownLatch(1); leftLatch = new CountDownLatch(1); @@ -797,23 +671,7 @@ public void testRestartNodeById() throws Exception { public void testRestartNodesByIds() throws Exception { joinedLatch = new CountDownLatch(3); - Collection res = - startNodes(ignite.cluster(), - maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), - false, 0, 16); - - assert res.size() == 3; - - res.forEach(t -> { - assert t.getHostName().equals(HOST); - - if (!t.isSuccess()) - throw new IgniteException(t.getError()); - }); - - assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); - - assert ignite.cluster().nodes().size() == 3; + startCheckNodes(); joinedLatch = new CountDownLatch(2); leftLatch = new CountDownLatch(2); @@ -895,6 +753,30 @@ private Collection> maps( return maps; } + /** + * @throws InterruptedException If failed. + */ + private void startCheckNodes() throws InterruptedException { + joinedLatch = new CountDownLatch(3); + + Collection res = + startNodes(ignite.cluster(), + maps(Collections.singleton(HOST), pwd, key, 3, U.getIgniteHome(), CFG_NO_ATTR, null), + false, 0, 16); + + assert res.size() == 3; + + res.forEach(t -> { + assert t.getHostName().equals(HOST); + + if (!t.isSuccess()) + throw new IgniteException(t.getError()); + }); + + assert joinedLatch.await(WAIT_TIMEOUT, MILLISECONDS); + + assert ignite.cluster().nodes().size() == 3; + } /** * @param cluster Cluster. @@ -921,4 +803,4 @@ private static String getProperty(String envName, String gridTestName) { return GridTestProperties.getProperty(gridTestName); } -} +} \ No newline at end of file From 3ed71fe9a12287fd468723d9d421e41c3c929aaf Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Fri, 9 Nov 2018 15:00:03 +0300 Subject: [PATCH 075/403] IGNITE-9870 GridDhtPartitionsFullMessage#prepareMarshal compression parallelization - Fixes #5330. Signed-off-by: Alexey Goncharuk --- .../configuration/IgniteConfiguration.java | 32 +- .../cache/CacheAffinitySharedManager.java | 14 +- .../processors/cache/GridCacheProcessor.java | 44 ++- .../GridDhtPartitionsExchangeFuture.java | 8 +- .../GridDhtPartitionsFullMessage.java | 214 +++++++------ .../GridCacheDatabaseSharedManager.java | 4 +- .../ignite/internal/util/IgniteUtils.java | 281 ++++++++++-------- .../util/lang/IgniteThrowableConsumer.java | 6 +- .../internal/util/IgniteUtilsSelfTest.java | 122 +++++++- .../IgniteConfigurationParityTest.cs | 3 +- 10 files changed, 483 insertions(+), 245 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java index e7ccaf556ccf7..a04fc53f681aa 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java @@ -21,6 +21,7 @@ import java.lang.management.ManagementFactory; import java.util.Map; import java.util.UUID; +import java.util.zip.Deflater; import javax.cache.configuration.Factory; import javax.cache.event.CacheEntryListener; import javax.cache.expiry.ExpiryPolicy; @@ -40,7 +41,6 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeTask; -import org.apache.ignite.spi.encryption.EncryptionSpi; import org.apache.ignite.events.Event; import org.apache.ignite.events.EventType; import org.apache.ignite.failure.FailureHandler; @@ -69,6 +69,7 @@ import org.apache.ignite.spi.deployment.local.LocalDeploymentSpi; import org.apache.ignite.spi.discovery.DiscoverySpi; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.encryption.EncryptionSpi; import org.apache.ignite.spi.eventstorage.EventStorageSpi; import org.apache.ignite.spi.eventstorage.NoopEventStorageSpi; import org.apache.ignite.spi.failover.FailoverSpi; @@ -119,6 +120,9 @@ public class IgniteConfiguration { /** Default maximum timeout to wait for network responses in milliseconds (value is {@code 5,000ms}). */ public static final long DFLT_NETWORK_TIMEOUT = 5000; + /** Default compression level for network messages (value is Deflater.BEST_SPEED. */ + public static final int DFLT_NETWORK_COMPRESSION = Deflater.BEST_SPEED; + /** Default interval between message send retries. */ public static final long DFLT_SEND_RETRY_DELAY = 1000; @@ -302,6 +306,9 @@ public class IgniteConfiguration { /** Maximum network requests timeout. */ private long netTimeout = DFLT_NETWORK_TIMEOUT; + /** Compression level for network binary messages. */ + private int netCompressionLevel = DFLT_NETWORK_COMPRESSION; + /** Interval between message send retries. */ private long sndRetryDelay = DFLT_SEND_RETRY_DELAY; @@ -1478,6 +1485,29 @@ public IgniteConfiguration setNetworkTimeout(long netTimeout) { return this; } + /** + * Compression level of internal network messages. + *

+ * If not provided, then default value + * Deflater.BEST_SPEED is used. + * + * @return Network messages default compression level. + */ + public int getNetworkCompressionLevel() { + return netCompressionLevel; + } + + /** + * Compression level for internal network messages. + *

+ * If not provided, then default value + * Deflater.BEST_SPEED is used. + * + */ + public void setNetworkCompressionLevel(int netCompressionLevel) { + this.netCompressionLevel = netCompressionLevel; + } + /** * Interval in milliseconds between message send retries. *

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 1283696393a87..936dc3eb81889 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -993,6 +993,8 @@ private void initAffinityOnCacheGroupsStart( initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut); } } + + return null; } ); } @@ -1255,7 +1257,11 @@ private void forAllRegisteredCacheGroups(IgniteInClosureX .collect(Collectors.toList()); try { - U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c::applyx); + U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, t -> { + c.applyx(t); + + return null; + }); } catch (IgniteCheckedException e) { throw new IgniteException("Failed to execute affinity operation on cache groups", e); @@ -1283,7 +1289,11 @@ private void forAllCacheGroups(boolean crd, IgniteInClosureX { + c.applyx(t); + + return null; + }); } catch (IgniteCheckedException e) { throw new IgniteException("Failed to execute affinity operation on cache groups", e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 27947dc691e4c..2e8dcf050226c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2067,19 +2067,23 @@ Map prepareStartCachesIfPossible(Collect */ private void prepareStartCaches( Collection startCacheInfos, - StartCacheFailHandler cacheStartFailHandler + StartCacheFailHandler cacheStartFailHandler ) throws IgniteCheckedException { if (!IGNITE_ALLOW_START_CACHES_IN_PARALLEL || startCacheInfos.size() <= 1) { for (StartCacheInfo startCacheInfo : startCacheInfos) { cacheStartFailHandler.handle( startCacheInfo, - cacheInfo -> prepareCacheStart( - cacheInfo.getCacheDescriptor().cacheConfiguration(), - cacheInfo.getCacheDescriptor(), - cacheInfo.getReqNearCfg(), - cacheInfo.getExchangeTopVer(), - cacheInfo.isDisabledAfterStart() - ) + cacheInfo -> { + prepareCacheStart( + cacheInfo.getCacheDescriptor().cacheConfiguration(), + cacheInfo.getCacheDescriptor(), + cacheInfo.getReqNearCfg(), + cacheInfo.getExchangeTopVer(), + cacheInfo.isDisabledAfterStart() + ); + + return null; + } ); context().exchange().exchangerUpdateHeartbeat(); @@ -2097,7 +2101,7 @@ private void prepareStartCaches( parallelismLvl, sharedCtx.kernalContext().getSystemExecutorService(), startCacheInfos, - startCacheInfo -> + startCacheInfo -> { cacheStartFailHandler.handle( startCacheInfo, cacheInfo -> { @@ -2111,8 +2115,13 @@ private void prepareStartCaches( cacheContexts.put(cacheInfo, cacheCtx); context().exchange().exchangerUpdateHeartbeat(); + + return null; } - ) + ); + + return null; + } ); /* @@ -2144,6 +2153,8 @@ private void prepareStartCaches( } context().exchange().exchangerUpdateHeartbeat(); + + return null; } ); } @@ -2152,7 +2163,7 @@ private void prepareStartCaches( parallelismLvl, sharedCtx.kernalContext().getSystemExecutorService(), cacheContexts.entrySet(), - cacheCtxEntry -> + cacheCtxEntry -> { cacheStartFailHandler.handle( cacheCtxEntry.getKey(), cacheInfo -> { @@ -2164,8 +2175,13 @@ private void prepareStartCaches( onCacheStarted(cacheCtxEntry.getValue()); context().exchange().exchangerUpdateHeartbeat(); + + return null; } - ) + ); + + return null; + } ); } } @@ -5468,7 +5484,7 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da * * @param Type of started data. */ - private static interface StartCacheFailHandler { + private static interface StartCacheFailHandler { /** * Handle of fail. * @@ -5476,7 +5492,7 @@ private static interface StartCacheFailHandler { * @param startCacheOperation Operation for start cache. * @throws IgniteCheckedException if failed. */ - void handle(T data, IgniteThrowableConsumer startCacheOperation) throws IgniteCheckedException; + void handle(T data, IgniteThrowableConsumer startCacheOperation) throws IgniteCheckedException; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 0636e90ff962d..5387180bef09c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3497,7 +3497,7 @@ private void validatePartitionsState() { || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE || grpCtx.config().getExpiryPolicyFactory() == null || SKIP_PARTITION_SIZE_VALIDATION) - return; + return null; try { validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs); @@ -3506,6 +3506,8 @@ private void validatePartitionsState() { log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage()); // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833 } + + return null; } ); } @@ -3538,6 +3540,8 @@ private void assignPartitionsStates() { assignPartitionSizes(top); else assignPartitionStates(top); + + return null; } ); } @@ -3994,6 +3998,8 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa null); } } + + return null; }); } catch (IgniteCheckedException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index a63ab70e28d53..4e895ed9d5353 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -19,8 +19,11 @@ import java.io.Externalizable; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -33,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; @@ -410,69 +414,75 @@ void setErrorsMap(Map errs) { (!F.isEmpty(errs) && errsBytes == null); if (marshal) { - byte[] partsBytes0 = null; - byte[] partCntrsBytes0 = null; - byte[] partCntrsBytes20 = null; - byte[] partHistSuppliersBytes0 = null; - byte[] partsToReloadBytes0 = null; - byte[] partsSizesBytes0 = null; - byte[] errsBytes0 = null; + // Reserve at least 2 threads for system operations. + int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2); + + Collection objectsToMarshall = new ArrayList<>(); + + if (!F.isEmpty(parts) && partsBytes == null) + objectsToMarshall.add(parts); + + if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null) + objectsToMarshall.add(partCntrs); + + if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null) + objectsToMarshall.add(partCntrs2); + + if (partHistSuppliers != null && partHistSuppliersBytes == null) + objectsToMarshall.add(partHistSuppliers); + + if (partsToReload != null && partsToReloadBytes == null) + objectsToMarshall.add(partsToReload); + + if (partsSizes != null && partsSizesBytes == null) + objectsToMarshall.add(partsSizes); + + if (!F.isEmpty(errs) && errsBytes == null) + objectsToMarshall.add(errs); + + Collection marshalled = U.doInParallel( + parallelismLvl, + ctx.kernalContext().getSystemExecutorService(), + objectsToMarshall, + new IgniteThrowableConsumer() { + @Override public byte[] accept(Object payload) throws IgniteCheckedException { + byte[] marshalled = U.marshal(ctx, payload); + + if(compress) + marshalled = U.zip(marshalled, ctx.gridConfig().getNetworkCompressionLevel()); + + return marshalled; + } + }); + + Iterator iterator = marshalled.iterator(); if (!F.isEmpty(parts) && partsBytes == null) - partsBytes0 = U.marshal(ctx, parts); + partsBytes = iterator.next(); if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null) - partCntrsBytes0 = U.marshal(ctx, partCntrs); + partCntrsBytes = iterator.next(); if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null) - partCntrsBytes20 = U.marshal(ctx, partCntrs2); + partCntrsBytes2 = iterator.next(); if (partHistSuppliers != null && partHistSuppliersBytes == null) - partHistSuppliersBytes0 = U.marshal(ctx, partHistSuppliers); + partHistSuppliersBytes = iterator.next(); if (partsToReload != null && partsToReloadBytes == null) - partsToReloadBytes0 = U.marshal(ctx, partsToReload); + partsToReloadBytes = iterator.next(); if (partsSizes != null && partsSizesBytes == null) - partsSizesBytes0 = U.marshal(ctx, partsSizes); + partsSizesBytes = iterator.next(); if (!F.isEmpty(errs) && errsBytes == null) - errsBytes0 = U.marshal(ctx, errs); + errsBytes = iterator.next(); if (compress) { - assert !compressed(); - - try { - byte[] partsBytesZip = U.zip(partsBytes0); - byte[] partCntrsBytesZip = U.zip(partCntrsBytes0); - byte[] partCntrsBytes2Zip = U.zip(partCntrsBytes20); - byte[] partHistSuppliersBytesZip = U.zip(partHistSuppliersBytes0); - byte[] partsToReloadBytesZip = U.zip(partsToReloadBytes0); - byte[] partsSizesBytesZip = U.zip(partsSizesBytes0); - byte[] exsBytesZip = U.zip(errsBytes0); - - partsBytes0 = partsBytesZip; - partCntrsBytes0 = partCntrsBytesZip; - partCntrsBytes20 = partCntrsBytes2Zip; - partHistSuppliersBytes0 = partHistSuppliersBytesZip; - partsToReloadBytes0 = partsToReloadBytesZip; - partsSizesBytes0 = partsSizesBytesZip; - errsBytes0 = exsBytesZip; - - compressed(true); - } - catch (IgniteCheckedException e) { - U.error(ctx.logger(getClass()), "Failed to compress partitions data: " + e, e); - } - } + assert !compressed() : "Unexpected compressed state"; - partsBytes = partsBytes0; - partCntrsBytes = partCntrsBytes0; - partCntrsBytes2 = partCntrsBytes20; - partHistSuppliersBytes = partHistSuppliersBytes0; - partsToReloadBytes = partsToReloadBytes0; - partsSizesBytes = partsSizesBytes0; - errsBytes = errsBytes0; + compressed(true); + } } } @@ -494,11 +504,51 @@ public void topologyVersion(AffinityTopologyVersion topVer) { @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException { super.finishUnmarshal(ctx, ldr); + ClassLoader classLoader = U.resolveClassLoader(ldr, ctx.gridConfig()); + + Collection objectsToUnmarshall = new ArrayList<>(); + + // Reserve at least 2 threads for system operations. + int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2); + + if (partsBytes != null && parts == null) + objectsToUnmarshall.add(partsBytes); + + if (partCntrsBytes != null && partCntrs == null) + objectsToUnmarshall.add(partCntrsBytes); + + if (partCntrsBytes2 != null && partCntrs2 == null) + objectsToUnmarshall.add(partCntrsBytes2); + + if (partHistSuppliersBytes != null && partHistSuppliers == null) + objectsToUnmarshall.add(partHistSuppliersBytes); + + if (partsToReloadBytes != null && partsToReload == null) + objectsToUnmarshall.add(partsToReloadBytes); + + if (partsSizesBytes != null && partsSizes == null) + objectsToUnmarshall.add(partsSizesBytes); + + if (errsBytes != null && errs == null) + objectsToUnmarshall.add(errsBytes); + + Collection unmarshalled = U.doInParallel( + parallelismLvl, + ctx.kernalContext().getSystemExecutorService(), + objectsToUnmarshall, + new IgniteThrowableConsumer() { + @Override public Object accept(byte[] binary) throws IgniteCheckedException { + return compressed() + ? U.unmarshalZip(ctx.marshaller(), binary, classLoader) + : U.unmarshal(ctx, binary, classLoader); + } + } + ); + + Iterator iterator = unmarshalled.iterator(); + if (partsBytes != null && parts == null) { - if (compressed()) - parts = U.unmarshalZip(ctx.marshaller(), partsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - parts = U.unmarshal(ctx, partsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); + parts = (Map)iterator.next(); if (dupPartsData != null) { assert parts != null; @@ -528,53 +578,41 @@ public void topologyVersion(AffinityTopologyVersion topVer) { } } - if (parts == null) - parts = new HashMap<>(); + if (partCntrsBytes != null && partCntrs == null) + partCntrs = (IgniteDhtPartitionCountersMap)iterator.next(); - if (partCntrsBytes != null && partCntrs == null) { - if (compressed()) - partCntrs = U.unmarshalZip(ctx.marshaller(), partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - partCntrs = U.unmarshal(ctx, partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - } + if (partCntrsBytes2 != null && partCntrs2 == null) + partCntrs2 = (IgniteDhtPartitionCountersMap2)iterator.next(); - if (partCntrsBytes2 != null && partCntrs2 == null) { - if (compressed()) - partCntrs2 = U.unmarshalZip(ctx.marshaller(), partCntrsBytes2, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - partCntrs2 = U.unmarshal(ctx, partCntrsBytes2, U.resolveClassLoader(ldr, ctx.gridConfig())); - } + if (partHistSuppliersBytes != null && partHistSuppliers == null) + partHistSuppliers = (IgniteDhtPartitionHistorySuppliersMap)iterator.next(); - if (partHistSuppliersBytes != null && partHistSuppliers == null) { - if (compressed()) - partHistSuppliers = U.unmarshalZip(ctx.marshaller(), partHistSuppliersBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - partHistSuppliers = U.unmarshal(ctx, partHistSuppliersBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - } + if (partsToReloadBytes != null && partsToReload == null) + partsToReload = (IgniteDhtPartitionsToReloadMap)iterator.next(); - if (partsToReloadBytes != null && partsToReload == null) { - if (compressed()) - partsToReload = U.unmarshalZip(ctx.marshaller(), partsToReloadBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - partsToReload = U.unmarshal(ctx, partsToReloadBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - } + if (partsSizesBytes != null && partsSizes == null) + partsSizes = (Map>)iterator.next(); - if (partsSizesBytes != null && partsSizes == null) { - if (compressed()) - partsSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - partsSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - } + if (errsBytes != null && errs == null) + errs = (Map)iterator.next(); + + if (parts == null) + parts = new HashMap<>(); if (partCntrs == null) partCntrs = new IgniteDhtPartitionCountersMap(); - if (errsBytes != null && errs == null) { - if (compressed()) - errs = U.unmarshalZip(ctx.marshaller(), errsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - else - errs = U.unmarshal(ctx, errsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); - } + if (partCntrs2 == null) + partCntrs2 = new IgniteDhtPartitionCountersMap2(); + + if(partHistSuppliers == null) + partHistSuppliers = new IgniteDhtPartitionHistorySuppliersMap(); + + if(partsToReload == null) + partsToReload = new IgniteDhtPartitionsToReloadMap(); + + if(partsSizes == null) + partsSizes = new HashMap<>(); if (errs == null) errs = new HashMap<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index f1a741d556c28..afba6bdb4d61e 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -1348,7 +1348,7 @@ private void shutdownCheckpointer(boolean cancel) { cctx.cache().cacheGroups(), cacheGroup -> { if (cacheGroup.isLocal()) - return; + return null; cctx.database().checkpointReadLock(); @@ -1361,6 +1361,8 @@ private void shutdownCheckpointer(boolean cancel) { finally { cctx.database().checkpointReadUnlock(); } + + return null; } ); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 127cc3a9226bc..dc72b37a964e6 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -17,6 +17,113 @@ package org.apache.ignite.internal.util; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteClientDisconnectedException; +import org.apache.ignite.IgniteDeploymentException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.IgniteInterruptedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.binary.BinaryRawWriter; +import org.apache.ignite.cluster.ClusterGroupEmptyException; +import org.apache.ignite.cluster.ClusterMetrics; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.cluster.ClusterTopologyException; +import org.apache.ignite.compute.ComputeTask; +import org.apache.ignite.compute.ComputeTaskCancelledException; +import org.apache.ignite.compute.ComputeTaskName; +import org.apache.ignite.compute.ComputeTaskTimeoutException; +import org.apache.ignite.configuration.AddressResolver; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.EventType; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException; +import org.apache.ignite.internal.IgniteDeploymentCheckedException; +import org.apache.ignite.internal.IgniteFutureCancelledCheckedException; +import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.IgniteNodeAttributes; +import org.apache.ignite.internal.cluster.ClusterGroupEmptyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException; +import org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException; +import org.apache.ignite.internal.events.DiscoveryCustomEvent; +import org.apache.ignite.internal.managers.communication.GridIoManager; +import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo; +import org.apache.ignite.internal.mxbean.IgniteStandardMXBean; +import org.apache.ignite.internal.processors.cache.CacheClassLoaderMarker; +import org.apache.ignite.internal.processors.cache.GridCacheAttributes; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cluster.BaselineTopology; +import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException; +import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException; +import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException; +import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.io.GridFilenameUtils; +import org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryNativeLoader; +import org.apache.ignite.internal.util.lang.GridClosureException; +import org.apache.ignite.internal.util.lang.GridPeerDeployAware; +import org.apache.ignite.internal.util.lang.GridTuple; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; +import org.apache.ignite.internal.util.typedef.C1; +import org.apache.ignite.internal.util.typedef.CI1; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.P1; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.lang.IgniteClosure; +import org.apache.ignite.lang.IgniteFutureCancelledException; +import org.apache.ignite.lang.IgniteFutureTimeoutException; +import org.apache.ignite.lang.IgniteOutClosure; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.lang.IgniteProductVersion; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.lifecycle.LifecycleAware; +import org.apache.ignite.marshaller.Marshaller; +import org.apache.ignite.plugin.PluginProvider; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageWriter; +import org.apache.ignite.spi.IgniteSpi; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.discovery.DiscoverySpi; +import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport; +import org.apache.ignite.transactions.TransactionDeadlockException; +import org.apache.ignite.transactions.TransactionHeuristicException; +import org.apache.ignite.transactions.TransactionOptimisticException; +import org.apache.ignite.transactions.TransactionRollbackException; +import org.apache.ignite.transactions.TransactionTimeoutException; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import sun.misc.Unsafe; + +import javax.management.DynamicMBean; +import javax.management.JMException; +import javax.management.MBeanRegistrationException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.naming.Context; +import javax.naming.NamingException; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -143,116 +250,11 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; -import javax.management.DynamicMBean; -import javax.management.JMException; -import javax.management.MBeanRegistrationException; -import javax.management.MBeanServer; -import javax.management.MalformedObjectNameException; -import javax.management.ObjectName; -import javax.naming.Context; -import javax.naming.NamingException; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteClientDisconnectedException; -import org.apache.ignite.IgniteDeploymentException; -import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteIllegalStateException; -import org.apache.ignite.IgniteInterruptedException; -import org.apache.ignite.IgniteLogger; -import org.apache.ignite.IgniteSystemProperties; -import org.apache.ignite.binary.BinaryRawReader; -import org.apache.ignite.binary.BinaryRawWriter; -import org.apache.ignite.cluster.ClusterGroupEmptyException; -import org.apache.ignite.cluster.ClusterMetrics; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.cluster.ClusterTopologyException; -import org.apache.ignite.compute.ComputeTask; -import org.apache.ignite.compute.ComputeTaskCancelledException; -import org.apache.ignite.compute.ComputeTaskName; -import org.apache.ignite.compute.ComputeTaskTimeoutException; -import org.apache.ignite.configuration.AddressResolver; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.events.EventType; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException; -import org.apache.ignite.internal.IgniteDeploymentCheckedException; -import org.apache.ignite.internal.IgniteFutureCancelledCheckedException; -import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.IgniteNodeAttributes; -import org.apache.ignite.internal.cluster.ClusterGroupEmptyCheckedException; -import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; -import org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException; -import org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException; -import org.apache.ignite.internal.events.DiscoveryCustomEvent; -import org.apache.ignite.internal.managers.communication.GridIoManager; -import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo; -import org.apache.ignite.internal.mxbean.IgniteStandardMXBean; -import org.apache.ignite.internal.processors.cache.CacheClassLoaderMarker; -import org.apache.ignite.internal.processors.cache.GridCacheAttributes; -import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cluster.BaselineTopology; -import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException; -import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException; -import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException; -import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; -import org.apache.ignite.internal.util.future.GridFutureAdapter; -import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.internal.util.io.GridFilenameUtils; -import org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryNativeLoader; -import org.apache.ignite.internal.util.lang.GridClosureException; -import org.apache.ignite.internal.util.lang.GridPeerDeployAware; -import org.apache.ignite.internal.util.lang.GridTuple; -import org.apache.ignite.internal.util.typedef.C1; -import org.apache.ignite.internal.util.typedef.CI1; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.G; -import org.apache.ignite.internal.util.typedef.P1; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.internal.util.typedef.internal.A; -import org.apache.ignite.internal.util.typedef.internal.SB; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.util.worker.GridWorker; -import org.apache.ignite.lang.IgniteBiTuple; -import org.apache.ignite.lang.IgniteClosure; -import org.apache.ignite.lang.IgniteFutureCancelledException; -import org.apache.ignite.lang.IgniteFutureTimeoutException; -import org.apache.ignite.lang.IgniteOutClosure; -import org.apache.ignite.lang.IgnitePredicate; -import org.apache.ignite.lang.IgniteProductVersion; -import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; -import org.apache.ignite.lang.IgniteUuid; -import org.apache.ignite.lifecycle.LifecycleAware; -import org.apache.ignite.marshaller.Marshaller; -import org.apache.ignite.plugin.PluginProvider; -import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.plugin.extensions.communication.MessageWriter; -import org.apache.ignite.spi.IgniteSpi; -import org.apache.ignite.spi.IgniteSpiException; -import org.apache.ignite.spi.discovery.DiscoverySpi; -import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport; -import org.apache.ignite.transactions.TransactionDeadlockException; -import org.apache.ignite.transactions.TransactionHeuristicException; -import org.apache.ignite.transactions.TransactionOptimisticException; -import org.apache.ignite.transactions.TransactionRollbackException; -import org.apache.ignite.transactions.TransactionTimeoutException; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import sun.misc.Unsafe; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISABLE_HOSTNAME_VERIFIER; import static org.apache.ignite.IgniteSystemProperties.IGNITE_HOME; @@ -10338,11 +10340,23 @@ public static void restoreOldIgniteName(@Nullable String oldName, @Nullable Stri } /** + * Zip binary payload using default compression. + * * @param bytes Byte array to compress. * @return Compressed bytes. * @throws IgniteCheckedException If failed. */ public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException { + return zip(bytes, Deflater.DEFAULT_COMPRESSION); + } + + /** + * @param bytes Byte array to compress. + * @param compressionLevel Level of compression to encode. + * @return Compressed bytes. + * @throws IgniteCheckedException If failed. + */ + public static byte[] zip(@Nullable byte[] bytes, int compressionLevel) throws IgniteCheckedException { try { if (bytes == null) return null; @@ -10350,6 +10364,8 @@ public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); try (ZipOutputStream zos = new ZipOutputStream(bos)) { + zos.setLevel(compressionLevel); + ZipEntry entry = new ZipEntry(""); try { @@ -10754,9 +10770,12 @@ public static ClusterNode randomServerNode(GridKernalContext ctx) { * @param Type of data. * @throws IgniteCheckedException if parallel execution was failed. */ - public static void doInParallel(ExecutorService executorSvc, Collection srcDatas, - IgniteThrowableConsumer operation) throws IgniteCheckedException, IgniteInterruptedCheckedException { - doInParallel(srcDatas.size(), executorSvc, srcDatas, operation); + public static Collection doInParallel( + ExecutorService executorSvc, + Collection srcDatas, + IgniteThrowableConsumer operation + ) throws IgniteCheckedException, IgniteInterruptedCheckedException { + return doInParallel(srcDatas.size(), executorSvc, srcDatas, operation); } /** @@ -10767,38 +10786,58 @@ public static void doInParallel(ExecutorService executorSvc, Collection s * @param srcDatas List of data for parallelization. * @param operation Logic for execution of on each item of data. * @param Type of data. + * @param Type of return value. * @throws IgniteCheckedException if parallel execution was failed. */ - public static void doInParallel( + public static Collection doInParallel( int parallelismLvl, ExecutorService executorSvc, Collection srcDatas, - IgniteThrowableConsumer operation + IgniteThrowableConsumer operation ) throws IgniteCheckedException, IgniteInterruptedCheckedException { + if(srcDatas.isEmpty()) + return Collections.emptyList(); + + int batchSize = srcDatas.size() / parallelismLvl; + + final int finalBatchSize = batchSize == 0 ? srcDatas.size() : batchSize; + List> batches = IntStream.range(0, parallelismLvl) - .mapToObj(i -> new ArrayList()) + .mapToObj(i -> new ArrayList(finalBatchSize)) .collect(Collectors.toList()); - int i = 0; + int batchIndex = 0; + + final int maxBatchIndex = batches.size() -1; - for (T src : srcDatas) - batches.get(i++ % parallelismLvl).add(src); + List currentBatch = batches.get(batchIndex); - List> consumerFutures = batches.stream() + for (T src : srcDatas) { + currentBatch.add(src); + + if(currentBatch.size() >= batchSize && batchIndex < maxBatchIndex) + currentBatch = batches.get(++batchIndex); + } + + List>> consumerFutures = batches.stream() .filter(batch -> !batch.isEmpty()) .map(batch -> executorSvc.submit(() -> { + Collection results = new ArrayList<>(batch.size()); + for (T item : batch) - operation.accept(item); + results.add(operation.accept(item)); - return null; + return results; })) .collect(Collectors.toList()); Throwable error =null; - for (Future future : consumerFutures) { + Collection results = new ArrayList<>(srcDatas.size()); + + for (Future> future : consumerFutures) { try { - future.get(); + results.addAll(future.get()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -10831,6 +10870,8 @@ public static void doInParallel( throw new IgniteCheckedException(error); } + + return results; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java index 46813a9a4cf29..55feed8cc2d33 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java @@ -25,13 +25,15 @@ * interfaces, {@code IgniteThrowableConsumer} is expected to operate via side-effects. * * @param Type of closure parameter. + * @param Type of result value. */ -public interface IgniteThrowableConsumer extends Serializable { +public interface IgniteThrowableConsumer extends Serializable { /** * Consumer body. * * @param e Consumer parameter. + * @return Result of consumer operation. * @throws IgniteCheckedException if body execution was failed. */ - public void accept(E e) throws IgniteCheckedException; + public R accept(E e) throws IgniteCheckedException; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java index ad678b6fc85b1..4d417de6f06d7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java @@ -45,9 +45,11 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterGroup; @@ -886,18 +888,26 @@ public void testIsOldestNodeVersionAtLeast() { public void testDoInParallel() throws Throwable { CyclicBarrier barrier = new CyclicBarrier(3); - IgniteUtils.doInParallel(3, - Executors.newFixedThreadPool(3), - Arrays.asList(1, 2, 3), - i -> { - try { - barrier.await(1, TimeUnit.SECONDS); - } - catch (Exception e) { - throw new IgniteCheckedException(e); + ExecutorService executorService = Executors.newFixedThreadPool(3); + + try { + IgniteUtils.doInParallel(3, + executorService, + Arrays.asList(1, 2, 3), + i -> { + try { + barrier.await(1, TimeUnit.SECONDS); + } + catch (Exception e) { + throw new IgniteCheckedException(e); + } + + return null; } - } - ); + ); + } finally { + executorService.shutdownNow(); + } } /** @@ -906,9 +916,11 @@ public void testDoInParallel() throws Throwable { public void testDoInParallelBatch() { CyclicBarrier barrier = new CyclicBarrier(3); + ExecutorService executorService = Executors.newFixedThreadPool(3); + try { IgniteUtils.doInParallel(2, - Executors.newFixedThreadPool(3), + executorService, Arrays.asList(1, 2, 3), i -> { try { @@ -917,6 +929,8 @@ public void testDoInParallelBatch() { catch (Exception e) { throw new IgniteCheckedException(e); } + + return null; } ); @@ -924,22 +938,98 @@ public void testDoInParallelBatch() { } catch (Exception e) { assertTrue(e.toString(), X.hasCause(e, TimeoutException.class)); + } finally { + executorService.shutdownNow(); } } + /** + * Test parallel execution in order. + */ + public void testDoInParallelResultsOrder() throws IgniteCheckedException { + ExecutorService executorService = Executors.newFixedThreadPool(4); + + try { + testOrder(executorService, 1, 1); + testOrder(executorService, 2, 1); + testOrder(executorService, 3, 1); + testOrder(executorService, 9, 1); + testOrder(executorService, 10, 1); + testOrder(executorService, 9999, 1); + + testOrder(executorService, 1, 2); + testOrder(executorService, 2, 2); + testOrder(executorService, 3, 2); + testOrder(executorService, 9, 2); + testOrder(executorService, 10, 2); + testOrder(executorService, 9999, 2); + + testOrder(executorService, 1, 3); + testOrder(executorService, 2, 3); + testOrder(executorService, 3, 3); + testOrder(executorService, 3, 3); + testOrder(executorService, 10, 3); + testOrder(executorService, 9999, 3); + + testOrder(executorService, 1, 4); + testOrder(executorService, 2, 4); + testOrder(executorService, 3, 4); + testOrder(executorService, 9, 4); + testOrder(executorService, 10, 4); + testOrder(executorService, 9999, 4); + } finally { + executorService.shutdownNow(); + } + } + + /** + * Template method to test parallel execution + * @param executorService ExecutorService. + * @param size Size. + * @param parallelism Parallelism. + * @throws IgniteCheckedException Exception. + */ + private void testOrder(ExecutorService executorService, int size, int parallelism) throws IgniteCheckedException { + List list = new ArrayList<>(); + for(int i = 0; i < size; i++) + list.add(i); + + Collection results = IgniteUtils.doInParallel( + parallelism, + executorService, + list, + i -> i * 2 + ); + + assertEquals(list.size(), results.size()); + + final int[] i = {0}; + results.forEach(new Consumer() { + @Override public void accept(Integer integer) { + assertEquals(2 * list.get(i[0]), integer.intValue()); + i[0]++; + } + }); + } + /** * */ public void testDoInParallelException() { String expectedException = "ExpectedException"; + ExecutorService executorService = Executors.newFixedThreadPool(1); + try { - IgniteUtils.doInParallel(3, - Executors.newFixedThreadPool(1), + IgniteUtils.doInParallel( + 1, + executorService, Arrays.asList(1, 2, 3), i -> { - if (i == 1) + if (Integer.valueOf(1).equals(i)) throw new IgniteCheckedException(expectedException); + + return null; } ); @@ -947,6 +1037,8 @@ public void testDoInParallelException() { } catch (IgniteCheckedException e) { assertEquals(expectedException, e.getMessage()); + } finally { + executorService.shutdownNow(); } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs index 71d2ba07a9511..a50d48a64a32c 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs @@ -81,7 +81,8 @@ public class IgniteConfigurationParityTest "TimeServerPortBase", "TimeServerPortRange", "IncludeProperties", - "isAutoActivationEnabled" // IGNITE-7301 + "isAutoActivationEnabled", // IGNITE-7301 + "NetworkCompressionLevel" }; /// From 41d3244ba1842ced06f98157e59c64aa91400101 Mon Sep 17 00:00:00 2001 From: Alexey Platonov Date: Fri, 9 Nov 2018 15:33:27 +0300 Subject: [PATCH 076/403] IGNITE-9930 Split ZK Discovery 2 suite into several configs - Fixes #5021. Signed-off-by: Alexey Goncharuk --- .../zk/ZookeeperDiscoverySpiTestSuite2.java | 46 +------------- .../zk/ZookeeperDiscoverySpiTestSuite3.java | 62 +++++++++++++++++++ .../zk/ZookeeperDiscoverySpiTestSuite4.java | 57 +++++++++++++++++ 3 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite3.java create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite4.java diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java index 3b8ddeefefe1a..f64c57edb1895 100644 --- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java @@ -18,26 +18,11 @@ package org.apache.ignite.spi.discovery.zk; import junit.framework.TestSuite; -import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster; -import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest; import org.apache.ignite.internal.IgniteClientReconnectCacheTest; -import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest; -import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedNodeRestartTxSelfTest; -import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedSequenceApiSelfTest; -import org.apache.ignite.internal.processors.cache.datastructures.replicated.GridCacheReplicatedSequenceApiSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryAtomicSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryTransactionalSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNodeRestartTest; -import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeFullApiSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedNodeRestartSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQuerySelfTest; -import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest; -import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest; -import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest; -import org.apache.ignite.p2p.GridP2PContinuousDeploymentSelfTest; +import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster; import org.apache.ignite.util.GridCommandHandlerTest; /** @@ -58,42 +43,13 @@ public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite"); - suite.addTestSuite(ZookeeperDiscoverySuitePreprocessorTest.class); - - suite.addTestSuite(GridCacheReplicatedNodeRestartSelfTest.class); suite.addTestSuite(GridCachePartitionedNodeRestartTest.class); - suite.addTestSuite(IgniteCacheEntryListenerWithZkDiscoAtomicTest.class); - - suite.addTestSuite(GridEventConsumeSelfTest.class); - suite.addTestSuite(IgniteClientReconnectCacheTest.class); - - suite.addTestSuite(IgniteCachePutRetryAtomicSelfTest.class); - suite.addTestSuite(IgniteCachePutRetryTransactionalSelfTest.class); - - suite.addTestSuite(ClusterNodeMetricsUpdateTest.class); - suite.addTestSuite(GridCachePartitionedMultiNodeFullApiSelfTest.class); suite.addTestSuite(GridCacheReplicatedMultiNodeFullApiSelfTest.class); - - suite.addTestSuite(GridCacheAtomicMultiNodeFullApiSelfTest.class); - suite.addTestSuite(GridCacheReplicatedAtomicMultiNodeFullApiSelfTest.class); - - suite.addTestSuite(GridCachePartitionedNodeRestartTxSelfTest.class); - suite.addTestSuite(IgniteClientDataStructuresTest.class); - suite.addTestSuite(GridCacheReplicatedSequenceApiSelfTest.class); - suite.addTestSuite(GridCachePartitionedSequenceApiSelfTest.class); - - suite.addTestSuite(IgniteCacheReplicatedQuerySelfTest.class); - - suite.addTestSuite(GridCacheAtomicMultiJvmFullApiSelfTest.class); - suite.addTestSuite(GridCachePartitionedMultiJvmFullApiSelfTest.class); - suite.addTestSuite(GridCommandHandlerTest.class); - suite.addTestSuite(GridP2PContinuousDeploymentSelfTest.class); - return suite; } } diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite3.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite3.java new file mode 100644 index 0000000000000..c988205085397 --- /dev/null +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite3.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.zk; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest; +import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedNodeRestartTxSelfTest; +import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedSequenceApiSelfTest; +import org.apache.ignite.internal.processors.cache.datastructures.replicated.GridCacheReplicatedSequenceApiSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedNodeRestartSelfTest; +import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest; +import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest; +import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest; +import org.apache.ignite.p2p.GridP2PContinuousDeploymentSelfTest; +import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster; + +/** + * Regular Ignite tests executed with {@link ZookeeperDiscoverySpi}. + */ +public class ZookeeperDiscoverySpiTestSuite3 extends ZookeeperDiscoverySpiAbstractTestSuite { + /** */ + private static TestingCluster testingCluster; + + /** + * @return Test suite. + * @throws Exception Thrown in case of the failure. + */ + public static TestSuite suite() throws Exception { + System.setProperty("H2_JDBC_CONNECTIONS", "500"); // For multi-jvm tests. + + initSuite(); + + TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite"); + + suite.addTestSuite(GridCacheReplicatedNodeRestartSelfTest.class); + suite.addTestSuite(GridEventConsumeSelfTest.class); + suite.addTestSuite(GridCachePartitionedNodeRestartTxSelfTest.class); + suite.addTestSuite(IgniteClientDataStructuresTest.class); + suite.addTestSuite(GridCacheReplicatedSequenceApiSelfTest.class); + suite.addTestSuite(GridCachePartitionedSequenceApiSelfTest.class); + suite.addTestSuite(GridCacheAtomicMultiJvmFullApiSelfTest.class); + suite.addTestSuite(GridCachePartitionedMultiJvmFullApiSelfTest.class); + suite.addTestSuite(GridP2PContinuousDeploymentSelfTest.class); + + return suite; + } +} diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite4.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite4.java new file mode 100644 index 0000000000000..cb8010ba62afb --- /dev/null +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite4.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.zk; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryTransactionalSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicMultiNodeFullApiSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQuerySelfTest; +import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster; + +/** + * Regular Ignite tests executed with {@link ZookeeperDiscoverySpi}. + */ +public class ZookeeperDiscoverySpiTestSuite4 extends ZookeeperDiscoverySpiAbstractTestSuite { + /** */ + private static TestingCluster testingCluster; + + /** + * @return Test suite. + * @throws Exception Thrown in case of the failure. + */ + public static TestSuite suite() throws Exception { + System.setProperty("H2_JDBC_CONNECTIONS", "500"); // For multi-jvm tests. + + initSuite(); + + TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite"); + + suite.addTestSuite(ZookeeperDiscoverySuitePreprocessorTest.class); + suite.addTestSuite(IgniteCachePutRetryAtomicSelfTest.class); + suite.addTestSuite(IgniteCachePutRetryTransactionalSelfTest.class); + suite.addTestSuite(ClusterNodeMetricsUpdateTest.class); + suite.addTestSuite(GridCacheAtomicMultiNodeFullApiSelfTest.class); + suite.addTestSuite(GridCacheReplicatedAtomicMultiNodeFullApiSelfTest.class); + suite.addTestSuite(IgniteCacheReplicatedQuerySelfTest.class); + + return suite; + } +} From 226d64f72a41ec677a8ee7ca6e494779cbadf0d3 Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Fri, 9 Nov 2018 17:57:17 +0300 Subject: [PATCH 077/403] IGNITE-9970 Added ability to set node ID for visor idle verify dump task - Fixes #5227. Signed-off-by: Alexey Goncharuk --- .../internal/commandline/CommandHandler.java | 73 ++++++++++++++++--- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 167a806f777e2..e37da813c6ede 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -18,6 +18,8 @@ package org.apache.ignite.internal.commandline; import java.io.Console; +import java.io.IOException; +import java.net.InetAddress; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -34,6 +36,7 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeTask; @@ -65,6 +68,7 @@ import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord; import org.apache.ignite.internal.processors.cache.verify.PartitionKey; import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2; +import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.SB; @@ -115,6 +119,7 @@ import org.apache.ignite.internal.visor.verify.VisorViewCacheTask; import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg; import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.plugin.security.SecurityCredentials; import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider; @@ -581,6 +586,35 @@ private R executeTask(GridClient client, Class> return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null); } + /** + * @param client Client. + * + * @return List of hosts. + */ + private Stream> listHosts(GridClient client) throws GridClientException { + return client.compute().nodes(GridClientNode::connectable).stream() + .flatMap(node -> Stream.concat( + node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(), + node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream() + ) + .map(addr -> new IgniteBiTuple<>(node, addr + ":" + node.tcpPort()))); + } + + /** + * @param client Client. + * + * @return List of hosts. + */ + private Stream>> listHostsByClientNode(GridClient client) throws GridClientException { + return client.compute().nodes(GridClientNode::connectable).stream() + .map(node -> new IgniteBiTuple<>(node, + Stream.concat( + node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(), + node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream() + ) + .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList()))); + } + /** * @param client Client * @param taskClsName Task class name. @@ -589,7 +623,12 @@ private R executeTask(GridClient client, Class> * @return Task result. * @throws GridClientException If failed to execute task. */ - private R executeTaskByNameOnNode(GridClient client, String taskClsName, Object taskArgs, UUID nodeId + @SuppressWarnings("unchecked") + private R executeTaskByNameOnNode( + GridClient client, + String taskClsName, + Object taskArgs, + UUID nodeId ) throws GridClientException { GridClientCompute compute = client.compute(); @@ -609,22 +648,34 @@ private R executeTaskByNameOnNode(GridClient client, String taskClsName, Obj GridClientNode node = null; if (nodeId == null) { - Collection nodes = compute.nodes(GridClientNode::connectable); - // Prefer node from connect string. - String origAddr = clientCfg.getServers().iterator().next(); + final String cfgAddr = clientCfg.getServers().iterator().next(); - for (GridClientNode clientNode : nodes) { - Iterator it = F.concat(clientNode.tcpAddresses().iterator(), clientNode.tcpHostNames().iterator()); + String[] parts = cfgAddr.split(":"); - while (it.hasNext()) { - if (origAddr.equals(it.next() + ":" + clientNode.tcpPort())) { - node = clientNode; + if (DFLT_HOST.equals(parts[0])) { + InetAddress addr; - break; - } + try { + addr = IgniteUtils.getLocalHost(); } + catch (IOException e) { + throw new GridClientException("Can't get localhost name.", e); + } + + if (addr.isLoopbackAddress()) + throw new GridClientException("Can't find localhost name."); + + String origAddr = addr.getHostName() + ":" + parts[1]; + + node = listHosts(client).filter(tuple -> origAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null); + + if (node == null) + node = listHostsByClientNode(client).filter(tuple -> tuple.get2().size() == 1 && cfgAddr.equals(tuple.get2().get(0))). + findFirst().map(IgniteBiTuple::get1).orElse(null); } + else + node = listHosts(client).filter(tuple -> cfgAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null); // Otherwise choose random node. if (node == null) From 01378d34c7896136d9a0738af22c5b0e837ae890 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Mon, 22 Oct 2018 16:01:28 +0300 Subject: [PATCH 078/403] IGNITE-9945 Entry start version should not omit datacenter ID - Fixes #5034. Signed-off-by: Alexey Goncharuk (cherry picked from commit 4353618a7501e2dada6b0027cadc2b9c6b069fde) --- .../cache/CacheConflictResolutionManager.java | 2 +- .../processors/cache/GridCacheMapEntry.java | 9 ++---- .../version/GridCacheVersionManager.java | 31 +++++++++++++++++-- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java index 6d65d828fc298..9790f754a8d5a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheConflictResolutionManager.java @@ -17,7 +17,7 @@ package org.apache.ignite.internal.processors.cache; -import org.apache.ignite.internal.processors.cache.version.*; +import org.apache.ignite.internal.processors.cache.version.CacheVersionConflictResolver; /** * Conflict resolver manager. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 9bb8aec1b3ba1..87de4b9d358c1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -73,7 +73,6 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; -import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx; import org.apache.ignite.internal.processors.dr.GridDrType; import org.apache.ignite.internal.processors.query.IgniteSQLException; @@ -117,10 +116,8 @@ import static org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome.REMOVE_NO_VAL; import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.compareIgnoreOpCounter; import static org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.RowData.NO_KEY; -import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.CONCURRENT_UPDATE; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.DUPLICATE_KEY; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.TRANSACTION_SERIALIZATION_ERROR; -import static org.apache.ignite.internal.processors.dr.GridDrType.DR_BACKUP; import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE; /** @@ -222,7 +219,7 @@ protected GridCacheMapEntry( this.cctx = cctx; this.listenerLock = cctx.continuousQueries().getListenerReadLock(); - ver = GridCacheVersionManager.START_VER; + ver = cctx.shared().versions().startVersion(); } /** @@ -332,7 +329,7 @@ protected void value(@Nullable CacheObject val) { * @return {@code True} if start version. */ public boolean isStartVersion() { - return ver == GridCacheVersionManager.START_VER; + return cctx.shared().versions().isStartVersion(ver); } /** {@inheritDoc} */ @@ -3292,7 +3289,7 @@ protected final boolean hasValueUnlocked() { GridCacheVersion currentVer = row != null ? row.version() : GridCacheMapEntry.this.ver; - boolean isStartVer = currentVer == GridCacheVersionManager.START_VER; + boolean isStartVer = cctx.shared().versions().isStartVersion(currentVer); if (cctx.group().persistenceEnabled()) { if (!isStartVer) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java index df8af4811b7b3..2a6526aa2a377 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java @@ -41,9 +41,6 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { /** */ public static final GridCacheVersion EVICT_VER = new GridCacheVersion(Integer.MAX_VALUE, 0, 0, 0); - /** */ - public static final GridCacheVersion START_VER = new GridCacheVersion(0, 0, 0, 0); - /** Timestamp used as base time for cache topology version (January 1, 2014). */ public static final long TOP_VER_BASE_TIME = 1388520000000L; @@ -56,6 +53,9 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { /** Current order for store operations. */ private final AtomicLong loadOrder = new AtomicLong(0); + /** Entry start version. */ + private GridCacheVersion startVer; + /** Last version. */ private volatile GridCacheVersion last; @@ -87,6 +87,8 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { @Override public void start0() throws IgniteCheckedException { last = new GridCacheVersion(0, order.get(), 0, dataCenterId); + startVer = new GridCacheVersion(0, 0, 0, dataCenterId); + cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_METRICS_UPDATED); } @@ -104,6 +106,8 @@ public void dataCenterId(byte dataCenterId) { this.dataCenterId = dataCenterId; last = new GridCacheVersion(0, order.get(), 0, dataCenterId); + + startVer = new GridCacheVersion(0, 0, 0, dataCenterId); } /** @@ -301,4 +305,25 @@ private GridCacheVersion next(long topVer, boolean addTime, boolean forLoad, byt public GridCacheVersion last() { return last; } + + /** + * Gets start version. + * + * @return Start version. + */ + public GridCacheVersion startVersion() { + assert startVer != null; + + return startVer; + } + + /** + * Check if given version is start version. + * + * @param ver Version. + * @return {@code True} if given version is start version. + */ + public boolean isStartVersion(GridCacheVersion ver) { + return startVer.equals(ver); + } } From 7c38beaf8cfaacdb4cdcd1cb02724343fd638f73 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Mon, 22 Oct 2018 16:10:47 +0300 Subject: [PATCH 079/403] IGNITE-9945 Entry start version should not omit datacenter ID. (cherry picked from commit e5a467272f88e5561a4cc30b510a5608a3b5e739) --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 2 +- .../internal/processors/cache/dr/GridCacheDrManager.java | 3 ++- .../internal/processors/cache/dr/GridOsCacheDrManager.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 6f38757300f0f..08fbbf90afd10 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2101,7 +2101,7 @@ public void finishMerged() { if (drCacheCtx.isDrEnabled()) { try { - drCacheCtx.dr().onExchange(res, exchId.isLeft()); + drCacheCtx.dr().onExchange(res, exchId.isLeft(), activateCluster()); } catch (IgniteCheckedException e) { U.error(log, "Failed to notify DR: " + e, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java index f2a4b30c4af82..33a52a11a03d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridCacheDrManager.java @@ -60,9 +60,10 @@ public void replicate(KeyCacheObject key, * * @param topVer Topology version. * @param left {@code True} if exchange has been caused by node leave. + * @param activate {@code True} if exchange has been caused by cluster activation. * @throws IgniteCheckedException If failed. */ - public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException; + public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException; /** * @return {@code True} is DR is enabled. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java index f3c1b23f7c7d6..425e79c536344 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/dr/GridOsCacheDrManager.java @@ -78,7 +78,7 @@ public class GridOsCacheDrManager implements GridCacheDrManager { } /** {@inheritDoc} */ - @Override public void onExchange(AffinityTopologyVersion topVer, boolean left) throws IgniteCheckedException { + @Override public void onExchange(AffinityTopologyVersion topVer, boolean left, boolean activate) throws IgniteCheckedException { // No-op. } From 6f737e3ba8eb2e52aa1148ab36069a90a9adfca1 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Mon, 12 Nov 2018 11:54:17 +0300 Subject: [PATCH 080/403] IGNITE-10085 Made compressed WAL archives more user-friendly - Fixes #5327. --- .../cache/persistence/wal/FileWriteAheadLogManager.java | 2 +- .../persistence/wal/FsyncModeFileWriteAheadLogManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 53ea14f7b0ad5..3bc47e733d536 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -2149,7 +2149,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip) try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) { zos.setLevel(dsCfg.getWalCompactionLevel()); - zos.putNextEntry(new ZipEntry("")); + zos.putNextEntry(new ZipEntry(nextSegment + ".wal")); ByteBuffer buf = ByteBuffer.allocate(HEADER_RECORD_SIZE); buf.order(ByteOrder.nativeOrder()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index c74c7eff8d34e..b1ad8ddca5a4b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -2083,7 +2083,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip) try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) { zos.setLevel(dsCfg.getWalCompactionLevel()); - zos.putNextEntry(new ZipEntry("")); + zos.putNextEntry(new ZipEntry(nextSegment + ".wal")); zos.write(prepareSerializerVersionBuffer(nextSegment, segmentSerializerVer, true).array()); From ef1f48ddcc4bdcbfa43fe29da54608a5b468d436 Mon Sep 17 00:00:00 2001 From: Alexey Platonov Date: Mon, 12 Nov 2018 16:45:40 +0300 Subject: [PATCH 081/403] IGNITE-9886 Split Continuous Query 1 into several configs - Fixes #4991. --- .../IgniteCacheQuerySelfTestSuite3.java | 116 ++++-------------- .../IgniteCacheQuerySelfTestSuite5.java | 73 +++++++++++ .../IgniteCacheQuerySelfTestSuite6.java | 73 +++++++++++ 3 files changed, 172 insertions(+), 90 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite5.java create mode 100644 modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite6.java diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java index 5dde4ccbb22c1..433aa2247edaf 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java @@ -18,59 +18,29 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchAckTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchForceServerModeAckTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFilterListenerTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryConcurrentPartitionUpdateTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryEventBufferTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryCounterPartitionedAtomicTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryCounterPartitionedTxTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryCounterReplicatedAtomicTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryCounterReplicatedTxTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryExecuteInPrimaryTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryAsyncFilterRandomOperationTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryFilterRandomOperationTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryLostPartitionTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryOperationFromCallbackTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryOperationP2PTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryOrderingEventTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryRandomOperationsTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryRandomOperationsTwoNodesTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverAtomicNearEnabledSelfSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerClientSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerFailoverTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerLocalSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerPartitionedSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerRandomOperationsTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerReplicatedSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationNearEnabledTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationStoreEnabledTest; -import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationTest; import org.apache.ignite.internal.processors.cache.query.continuous.ClientReconnectContinuousQueryTest; -import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryMarshallerTest; -import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryPeerClassLoadingTest; import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryReassignmentTest; -import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryRemoteFilterMissingInClassPathSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicNearEnabledSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicP2PDisabledSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryConcurrentTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryLocalAtomicSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryLocalSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryMultiNodesFilteringTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryNodesFilteringTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionAtomicOneNodeTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionTxOneNodeTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionedOnlySelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionedP2PDisabledSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionedSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedAtomicOneNodeTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedAtomicSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedP2PDisabledSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedTxOneNodeTest; -import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryTxSelfTest; -import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryBackupQueueTest; import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientReconnectTest; -import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientTest; import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientTxReconnectTest; -import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryImmutableEntryTest; import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryNoUnsubscribeTest; +import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryReconnectTest; /** * Test suite for cache queries. @@ -81,76 +51,42 @@ public class IgniteCacheQuerySelfTestSuite3 extends TestSuite { * @throws Exception If failed. */ public static TestSuite suite() throws Exception { - TestSuite suite = new TestSuite("Ignite Cache Queries Test Suite 3"); + TestSuite suite = new TestSuite("Ignite Cache Continuous Queries Test Suite"); - // Continuous queries. - suite.addTestSuite(GridCacheContinuousQueryLocalSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryLocalAtomicSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryReplicatedSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryReplicatedAtomicSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryReplicatedP2PDisabledSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryPartitionedSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryPartitionedOnlySelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryPartitionedP2PDisabledSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryTxSelfTest.class); + // Continuous queries 1. + suite.addTestSuite(GridCacheContinuousQueryNodesFilteringTest.class); + suite.addTestSuite(GridCacheContinuousQueryPartitionTxOneNodeTest.class); + suite.addTestSuite(CacheContinuousWithTransformerReplicatedSelfTest.class); + suite.addTestSuite(CacheContinuousQueryExecuteInPrimaryTest.class); + suite.addTestSuite(CacheContinuousWithTransformerClientSelfTest.class); + suite.addTestSuite(ClientReconnectContinuousQueryTest.class); + suite.addTestSuite(IgniteCacheContinuousQueryNoUnsubscribeTest.class); + suite.addTestSuite(IgniteCacheContinuousQueryClientTxReconnectTest.class); + suite.addTestSuite(IgniteCacheContinuousQueryClientReconnectTest.class); suite.addTestSuite(GridCacheContinuousQueryAtomicSelfTest.class); suite.addTestSuite(GridCacheContinuousQueryAtomicNearEnabledSelfTest.class); - suite.addTestSuite(GridCacheContinuousQueryAtomicP2PDisabledSelfTest.class); - - suite.addTestSuite(GridCacheContinuousQueryReplicatedTxOneNodeTest.class); - suite.addTestSuite(GridCacheContinuousQueryReplicatedAtomicOneNodeTest.class); suite.addTestSuite(GridCacheContinuousQueryPartitionTxOneNodeTest.class); - suite.addTestSuite(GridCacheContinuousQueryPartitionAtomicOneNodeTest.class); - - suite.addTestSuite(IgniteCacheContinuousQueryClientTest.class); suite.addTestSuite(IgniteCacheContinuousQueryClientReconnectTest.class); suite.addTestSuite(IgniteCacheContinuousQueryClientTxReconnectTest.class); - - suite.addTestSuite(CacheContinuousQueryRandomOperationsTest.class); - suite.addTestSuite(CacheContinuousQueryRandomOperationsTwoNodesTest.class); - suite.addTestSuite(GridCacheContinuousQueryConcurrentTest.class); - - suite.addTestSuite(CacheContinuousQueryAsyncFilterListenerTest.class); - suite.addTestSuite(CacheContinuousQueryFactoryFilterRandomOperationTest.class); + suite.addTestSuite(GridCacheContinuousQueryReplicatedSelfTest.class); suite.addTestSuite(CacheContinuousQueryFactoryAsyncFilterRandomOperationTest.class); - suite.addTestSuite(CacheContinuousQueryOrderingEventTest.class); - suite.addTestSuite(CacheContinuousQueryOperationFromCallbackTest.class); - suite.addTestSuite(CacheContinuousQueryOperationP2PTest.class); - suite.addTestSuite(CacheContinuousBatchAckTest.class); + suite.addTestSuite(GridCacheContinuousQueryPartitionedSelfTest.class); suite.addTestSuite(CacheContinuousBatchForceServerModeAckTest.class); suite.addTestSuite(CacheContinuousQueryExecuteInPrimaryTest.class); - suite.addTestSuite(CacheContinuousQueryLostPartitionTest.class); - suite.addTestSuite(ContinuousQueryRemoteFilterMissingInClassPathSelfTest.class); suite.addTestSuite(GridCacheContinuousQueryNodesFilteringTest.class); - suite.addTestSuite(GridCacheContinuousQueryMultiNodesFilteringTest.class); - suite.addTestSuite(IgniteCacheContinuousQueryImmutableEntryTest.class); - suite.addTestSuite(CacheKeepBinaryIterationTest.class); - suite.addTestSuite(CacheKeepBinaryIterationStoreEnabledTest.class); - suite.addTestSuite(CacheKeepBinaryIterationNearEnabledTest.class); - suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class); suite.addTestSuite(IgniteCacheContinuousQueryNoUnsubscribeTest.class); suite.addTestSuite(ClientReconnectContinuousQueryTest.class); - suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class); - suite.addTestSuite(ContinuousQueryMarshallerTest.class); suite.addTestSuite(ContinuousQueryReassignmentTest.class); - suite.addTestSuite(CacheContinuousQueryConcurrentPartitionUpdateTest.class); - suite.addTestSuite(CacheContinuousQueryEventBufferTest.class); - - suite.addTestSuite(CacheContinuousWithTransformerReplicatedSelfTest.class); - suite.addTestSuite(CacheContinuousWithTransformerLocalSelfTest.class); - suite.addTestSuite(CacheContinuousWithTransformerPartitionedSelfTest.class); - suite.addTestSuite(CacheContinuousWithTransformerClientSelfTest.class); - suite.addTestSuite(CacheContinuousWithTransformerFailoverTest.class); - suite.addTestSuite(CacheContinuousWithTransformerRandomOperationsTest.class); + suite.addTestSuite(CacheContinuousQueryFactoryAsyncFilterRandomOperationTest.class); - //suite.addTestSuite(CacheContinuousQueryCounterPartitionedAtomicTest.class); - //suite.addTestSuite(CacheContinuousQueryCounterPartitionedTxTest.class); - //suite.addTestSuite(CacheContinuousQueryCounterReplicatedAtomicTest.class); - //suite.addTestSuite(CacheContinuousQueryCounterReplicatedTxTest.class); - //suite.addTestSuite(CacheContinuousQueryFailoverAtomicNearEnabledSelfSelfTest.class); + suite.addTestSuite(CacheContinuousQueryCounterPartitionedAtomicTest.class); + suite.addTestSuite(CacheContinuousQueryCounterPartitionedTxTest.class); + suite.addTestSuite(CacheContinuousQueryCounterReplicatedAtomicTest.class); + suite.addTestSuite(CacheContinuousQueryCounterReplicatedTxTest.class); + suite.addTestSuite(CacheContinuousQueryFailoverAtomicNearEnabledSelfSelfTest.class); - //suite.addTestSuite(IgniteCacheContinuousQueryReconnectTest.class); + suite.addTestSuite(IgniteCacheContinuousQueryReconnectTest.class); return suite; } diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite5.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite5.java new file mode 100644 index 0000000000000..6fb542e2c8e2f --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite5.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryEventBufferTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFactoryFilterRandomOperationTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryLostPartitionTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryOperationFromCallbackTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryRandomOperationsTwoNodesTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerFailoverTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerLocalSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryPeerClassLoadingTest; +import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryRemoteFilterMissingInClassPathSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicP2PDisabledSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryConcurrentTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryLocalSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionedP2PDisabledSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedP2PDisabledSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedTxOneNodeTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryTxSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryBackupQueueTest; +import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryImmutableEntryTest; + +/** + * Test suite for cache queries. + */ +public class IgniteCacheQuerySelfTestSuite5 extends TestSuite { + /** + * @return Test suite. + * @throws Exception If failed. + */ + public static TestSuite suite() throws Exception { + TestSuite suite = new TestSuite("Ignite Cache Continuous Queries Test Suite 2"); + + // Continuous queries 2. + suite.addTestSuite(IgniteCacheContinuousQueryImmutableEntryTest.class); + suite.addTestSuite(CacheContinuousWithTransformerLocalSelfTest.class); + suite.addTestSuite(CacheContinuousQueryEventBufferTest.class); + suite.addTestSuite(GridCacheContinuousQueryReplicatedTxOneNodeTest.class); + suite.addTestSuite(GridCacheContinuousQueryLocalSelfTest.class); + suite.addTestSuite(CacheContinuousWithTransformerFailoverTest.class); + suite.addTestSuite(ContinuousQueryRemoteFilterMissingInClassPathSelfTest.class); + suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class); + suite.addTestSuite(GridCacheContinuousQueryAtomicP2PDisabledSelfTest.class); + suite.addTestSuite(GridCacheContinuousQueryTxSelfTest.class); + suite.addTestSuite(GridCacheContinuousQueryReplicatedP2PDisabledSelfTest.class); + suite.addTestSuite(GridCacheContinuousQueryPartitionedP2PDisabledSelfTest.class); + suite.addTestSuite(CacheContinuousQueryLostPartitionTest.class); + suite.addTestSuite(GridCacheContinuousQueryConcurrentTest.class); + suite.addTestSuite(CacheContinuousQueryRandomOperationsTwoNodesTest.class); + suite.addTestSuite(IgniteCacheContinuousQueryBackupQueueTest.class); + suite.addTestSuite(CacheContinuousQueryOperationFromCallbackTest.class); + suite.addTestSuite(CacheContinuousQueryFactoryFilterRandomOperationTest.class); + + return suite; + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite6.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite6.java new file mode 100644 index 0000000000000..9727b181507ba --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite6.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousBatchAckTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFilterListenerTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryOperationP2PTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryOrderingEventTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryRandomOperationsTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerPartitionedSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousWithTransformerRandomOperationsTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationNearEnabledTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationStoreEnabledTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationTest; +import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryMarshallerTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryLocalAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryMultiNodesFilteringTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionAtomicOneNodeTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryPartitionedOnlySelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedAtomicOneNodeTest; +import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientTest; + +/** + * Test suite for cache queries. + */ +public class IgniteCacheQuerySelfTestSuite6 extends TestSuite { + /** + * @return Test suite. + * @throws Exception If failed. + */ + public static TestSuite suite() throws Exception { + TestSuite suite = new TestSuite("Ignite Cache Continuous Queries Test Suite 3"); + + // Continuous queries 3. + suite.addTestSuite(GridCacheContinuousQueryPartitionAtomicOneNodeTest.class); + suite.addTestSuite(CacheContinuousWithTransformerPartitionedSelfTest.class); + suite.addTestSuite(GridCacheContinuousQueryLocalAtomicSelfTest.class); + suite.addTestSuite(GridCacheContinuousQueryReplicatedAtomicOneNodeTest.class); + suite.addTestSuite(ContinuousQueryMarshallerTest.class); + suite.addTestSuite(GridCacheContinuousQueryReplicatedAtomicSelfTest.class); + suite.addTestSuite(CacheKeepBinaryIterationTest.class); + suite.addTestSuite(GridCacheContinuousQueryMultiNodesFilteringTest.class); + suite.addTestSuite(CacheKeepBinaryIterationStoreEnabledTest.class); + suite.addTestSuite(CacheKeepBinaryIterationNearEnabledTest.class); + suite.addTestSuite(GridCacheContinuousQueryPartitionedOnlySelfTest.class); + suite.addTestSuite(CacheContinuousQueryOperationP2PTest.class); + suite.addTestSuite(CacheContinuousBatchAckTest.class); + suite.addTestSuite(CacheContinuousQueryOrderingEventTest.class); + suite.addTestSuite(IgniteCacheContinuousQueryClientTest.class); + suite.addTestSuite(CacheContinuousQueryAsyncFilterListenerTest.class); + suite.addTestSuite(CacheContinuousWithTransformerRandomOperationsTest.class); + suite.addTestSuite(CacheContinuousQueryRandomOperationsTest.class); + + return suite; + } +} From 30b1993226e9a643603b03cd2421534214a7f85d Mon Sep 17 00:00:00 2001 From: Dmitrii Ryabov Date: Mon, 12 Nov 2018 18:27:02 +0300 Subject: [PATCH 082/403] IGNITE-10188 Remove persistence tests restrictions for TC - Fixes #5351. Signed-off-by: Alexey Goncharuk --- .../junits/GridAbstractTest.java | 43 +------------------ .../testsuites/IgniteBasicTestSuite.java | 3 -- ...niteBinaryObjectsComputeGridTestSuite.java | 3 -- .../IgniteCacheFullApiSelfTestSuite.java | 2 - .../testsuites/IgniteCacheTestSuite.java | 6 +-- .../testsuites/IgniteCacheTestSuite2.java | 4 -- .../testsuites/IgniteCacheTestSuite3.java | 3 -- .../testsuites/IgniteCacheTestSuite4.java | 5 +-- .../testsuites/IgniteCacheTestSuite5.java | 3 -- .../testsuites/IgniteCacheTestSuite6.java | 3 -- .../testsuites/IgniteCacheTestSuite8.java | 3 -- .../testsuites/IgniteCacheTestSuite9.java | 3 -- .../IgniteCacheWithIndexingTestSuite.java | 3 -- 13 files changed, 4 insertions(+), 80 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index 0195a812d6f9d..064080bcd5e5a 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -140,9 +140,6 @@ "JUnitTestCaseWithNonTrivialConstructors" }) public abstract class GridAbstractTest extends TestCase { - /** Persistence in tests allowed property. */ - public static final String PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY = "PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY"; - /************************************************************** * DO NOT REMOVE TRANSIENT - THIS OBJECT MIGHT BE TRANSFERRED * * TO ANOTHER NODE. * @@ -194,11 +191,8 @@ public abstract class GridAbstractTest extends TestCase { /** Number of tests. */ private int testCnt; - /** - * - */ - private static final boolean PERSISTENCE_ALLOWED = - IgniteSystemProperties.getBoolean(PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, true); + /** Lazily initialized current test method. */ + private volatile Method currTestMtd; /** * @@ -883,9 +877,6 @@ private void validateConfiguration(IgniteConfiguration cfg) { */ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, GridSpringResourceContext ctx) throws Exception { - - checkConfiguration(cfg); - if (!isRemoteJvm(igniteInstanceName)) { IgniteUtils.setCurrentIgniteName(igniteInstanceName); @@ -930,30 +921,6 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G return startRemoteGrid(igniteInstanceName, null, ctx); } - /** - * @param cfg Config. - */ - protected void checkConfiguration(IgniteConfiguration cfg) { - if (cfg == null) - return; - - if (!PERSISTENCE_ALLOWED) { - String errorMsg = "PERSISTENCE IS NOT ALLOWED IN THIS SUITE, PUT YOUR TEST TO ANOTHER ONE!"; - - DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration(); - - if (dsCfg != null) { - assertFalse(errorMsg, dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled()); - - DataRegionConfiguration[] dataRegionConfigurations = dsCfg.getDataRegionConfigurations(); - - if (dataRegionConfigurations != null) - for (DataRegionConfiguration dataRegionConfiguration : dataRegionConfigurations) - assertFalse(errorMsg, dataRegionConfiguration.isPersistenceEnabled()); - } - } - } - /** * Starts new grid at another JVM with given name. * @@ -1050,8 +1017,6 @@ protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration if (cfg == null) cfg = optimize(getConfiguration(igniteInstanceName)); - checkConfiguration(cfg); - if (locNode != null) { DiscoverySpi discoverySpi = locNode.configuration().getDiscoverySpi(); @@ -1090,8 +1055,6 @@ protected List additionalRemoteJvmArgs() { * @throws IgniteCheckedException On error. */ protected IgniteConfiguration optimize(IgniteConfiguration cfg) throws IgniteCheckedException { - checkConfiguration(cfg); - if (cfg.getLocalHost() == null) { if (cfg.getDiscoverySpi() instanceof TcpDiscoverySpi) { cfg.setLocalHost("127.0.0.1"); @@ -1706,8 +1669,6 @@ protected IgniteConfiguration getConfiguration(String igniteInstanceName, Ignite cfg.setFailureHandler(getFailureHandler(igniteInstanceName)); - checkConfiguration(cfg); - return cfg; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index ac2bed36d3a58..0a76ee3f794a9 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -85,7 +85,6 @@ import org.apache.ignite.spi.GridSpiLocalHostInjectionTest; import org.apache.ignite.startup.properties.NotStringSystemPropertyTest; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.GridAbstractTest; import org.apache.ignite.testframework.test.ConfigVariationsTestSuiteBuilderTest; import org.apache.ignite.testframework.test.ParametersTest; import org.apache.ignite.testframework.test.VariationsIteratorTest; @@ -110,8 +109,6 @@ public static TestSuite suite() throws Exception { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite(@Nullable final Set ignoredTests) throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("Ignite Basic Test Suite"); suite.addTest(IgniteMarshallerSelfTestSuite.suite(ignoredTests)); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java index 4820d455683f5..8798db9b1d667 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java @@ -19,7 +19,6 @@ import junit.framework.TestSuite; import org.apache.ignite.internal.GridComputationBinarylizableClosuresSelfTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * @@ -30,8 +29,6 @@ public class IgniteBinaryObjectsComputeGridTestSuite { * @throws Exception If failed. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = IgniteComputeGridTestSuite.suite(); suite.addTestSuite(GridComputationBinarylizableClosuresSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java index 7d40a6ad4243a..c73209147a775 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java @@ -90,8 +90,6 @@ public class IgniteCacheFullApiSelfTestSuite extends TestSuite { * @throws Exception If failed. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("Cache Full API Test Suite"); // One node. diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java index d6c1f96a56b97..394624d5c5724 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Set; import junit.framework.TestSuite; import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest; import org.apache.ignite.cache.IgniteWarmupClosureSelfTest; @@ -175,9 +176,6 @@ import org.apache.ignite.internal.processors.datastreamer.DataStreamerTimeoutTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamerUpdateAfterLoadTest; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.GridAbstractTest; - -import java.util.Set; /** * Test suite. @@ -197,8 +195,6 @@ public static TestSuite suite() throws Exception { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite(Set ignoredTests) throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite"); suite.addTestSuite(IgniteCacheEntryListenerAtomicTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index b8eb276af589c..75b32fcf42ec9 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -18,7 +18,6 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; - import org.apache.ignite.cache.affinity.rendezvous.ClusterNodeAttributeAffinityBackupFilterSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionExcludeNeighborsSelfTest; @@ -169,7 +168,6 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxTimeoutSelfTest; import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest; import org.apache.ignite.internal.processors.continuous.IgniteNoCustomEventsOnNodeStart; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -180,8 +178,6 @@ public class IgniteCacheTestSuite2 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 2"); // Local cache. diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java index 188a035646342..6c59c4c19a485 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java @@ -83,7 +83,6 @@ import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadStartStopEventsSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheDaemonNodeLocalSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalByteArrayValuesSelfTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -94,8 +93,6 @@ public class IgniteCacheTestSuite3 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 3"); suite.addTestSuite(IgniteCacheGroupsTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java index cb4be142db58a..8ea329ee0c0ce 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java @@ -97,12 +97,12 @@ import org.apache.ignite.internal.processors.cache.IgniteStartCacheInTransactionSelfTest; import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest; import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest; -import org.apache.ignite.internal.processors.cache.distributed.CacheResultIsNotNullOnPartitionLossTest; import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest; import org.apache.ignite.internal.processors.cache.distributed.CacheDiscoveryDataConcurrentJoinTest; import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.CacheGroupsPreloadTest; import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheResultIsNotNullOnPartitionLossTest; import org.apache.ignite.internal.processors.cache.distributed.CacheStartOnJoinTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutTest; @@ -153,7 +153,6 @@ import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryPartitionedTransactionalSelfTest; import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedAtomicSelfTest; import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedTransactionalSelfTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -164,8 +163,6 @@ public class IgniteCacheTestSuite4 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 4"); // Multi node update. diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index a583317824b6f..109391da98c05 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -48,7 +48,6 @@ import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest; import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -59,8 +58,6 @@ public class IgniteCacheTestSuite5 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 5"); suite.addTestSuite(CacheSerializableTransactionsTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index bd7236ebe9f9b..18ae7f8e8791e 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -50,7 +50,6 @@ import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest; import org.apache.ignite.internal.processors.cache.transactions.TxStateChangeEventTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -61,8 +60,6 @@ public class IgniteCacheTestSuite6 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 6"); suite.addTestSuite(CachePartitionStateTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java index 086c567dac0ea..8ba955efe3a49 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java @@ -25,7 +25,6 @@ import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -36,8 +35,6 @@ public class IgniteCacheTestSuite8 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 8"); // Cache metrics. diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index 4b47b417218d2..b8da01e4e9a08 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -27,7 +27,6 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; import org.apache.ignite.internal.processors.cache.transactions.TxDataConsistencyOnCommitFailureTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Test suite. @@ -38,8 +37,6 @@ public class IgniteCacheTestSuite9 extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("IgniteCache Test Suite part 9"); suite.addTestSuite(IgniteCacheGetCustomCollectionsSelfTest.class); diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index 8517ebbf2a503..ab79ce34ac6ea 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -42,7 +42,6 @@ import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest; import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest; import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest; -import org.apache.ignite.testframework.junits.GridAbstractTest; /** * Cache tests using indexing. @@ -53,8 +52,6 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite { * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() throws Exception { - System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false"); - TestSuite suite = new TestSuite("Ignite Cache With Indexing Test Suite"); suite.addTestSuite(InlineIndexHelperTest.class); From 5c23c982bbece9507c0d01f9dae7a1465f30efed Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Tue, 13 Nov 2018 11:26:36 +0300 Subject: [PATCH 083/403] IGNITE-10134 Add exception handling for job reduce results - Fixes #5250. Signed-off-by: Alexey Goncharuk --- .../VisorCacheConfigurationCollectorTask.java | 40 +++++++++++ .../ignite/util/GridCommandHandlerTest.java | 72 +++++++++++++++---- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java index fd224a893eb39..418eb1b57b81b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java @@ -17,9 +17,15 @@ package org.apache.ignite.internal.visor.cache; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import org.apache.ignite.IgniteException; +import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.jetbrains.annotations.Nullable; /** * Task that collect cache metrics from all nodes. @@ -34,4 +40,38 @@ public class VisorCacheConfigurationCollectorTask @Override protected VisorCacheConfigurationCollectorJob job(VisorCacheConfigurationCollectorTaskArg arg) { return new VisorCacheConfigurationCollectorJob(arg, debug); } + + /** {@inheritDoc} */ + @Override protected @Nullable Map reduce0( + List results + ) throws IgniteException { + if (results == null) + return null; + + Map map = new HashMap<>(); + + List resultsExceptions = null; + + for (ComputeJobResult res : results) { + if (res.getException() == null) + map.putAll(res.getData()); + else { + if (resultsExceptions == null) + resultsExceptions = new ArrayList<>(results.size()); + + resultsExceptions.add(new IgniteException("Job failed on node: " + res.getNode().id(), res.getException())); + } + } + + if (resultsExceptions != null) { + IgniteException e = new IgniteException("Reduce failed because of job failed on some nodes"); + + for (Exception ex : resultsExceptions) + e.addSuppressed(ex); + + throw e; + } + + return map; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6cbbb3fb2cbce..6edde596e6da3 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -1264,29 +1264,71 @@ public void testCacheAffinity() throws Exception { /** */ public void testCacheConfigNoOutputFormat() throws Exception { - testCacheConfig(null); + testCacheConfig(null, 1, 1); } /** */ - public void testCacheConfigSingleLineOutputFormat() throws Exception { - testCacheConfig(SINGLE_LINE); + public void testCacheConfigSingleLineOutputFormatSingleNodeSignleCache() throws Exception { + testCacheConfigSingleLineOutputFormat(1, 1); } /** */ - public void testCacheConfigMultiLineOutputFormat() throws Exception { - testCacheConfig(MULTI_LINE); + public void testCacheConfigSingleLineOutputFormatTwoNodeSignleCache() throws Exception { + testCacheConfigSingleLineOutputFormat(2, 1); } /** */ - private void testCacheConfig(OutputFormat outputFormat) throws Exception { - Ignite ignite = startGrid(); + public void testCacheConfigSingleLineOutputFormatTwoNodeManyCaches() throws Exception { + testCacheConfigSingleLineOutputFormat(2, 100); + } + + /** */ + public void testCacheConfigMultiLineOutputFormatSingleNodeSingleCache() throws Exception { + testCacheConfigMultiLineOutputFormat(1, 1); + } + + /** */ + public void testCacheConfigMultiLineOutputFormatTwoNodeSingleCache() throws Exception { + testCacheConfigMultiLineOutputFormat(2, 1); + } + + /** */ + public void testCacheConfigMultiLineOutputFormatTwoNodeManyCaches() throws Exception { + testCacheConfigMultiLineOutputFormat(2, 100); + } + + /** */ + private void testCacheConfigSingleLineOutputFormat(int nodesCnt, int cachesCnt) throws Exception { + testCacheConfig(SINGLE_LINE, nodesCnt, cachesCnt); + } + + /** */ + private void testCacheConfigMultiLineOutputFormat(int nodesCnt, int cachesCnt) throws Exception { + testCacheConfig(MULTI_LINE, nodesCnt, cachesCnt); + } + + /** */ + private void testCacheConfig(OutputFormat outputFormat, int nodesCnt, int cachesCnt) throws Exception { + assertTrue("Invalid number of nodes or caches", nodesCnt > 0 && cachesCnt > 0); + + Ignite ignite = startGrid(nodesCnt); ignite.cluster().active(true); - IgniteCache cache1 = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); + List ccfgs = new ArrayList<>(cachesCnt); + + for (int i = 0; i < cachesCnt; i++) { + ccfgs.add( + new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME + i) + ); + } + + ignite.createCaches(ccfgs); + + IgniteCache cache1 = ignite.cache(DEFAULT_CACHE_NAME + 0); for (int i = 0; i < 100; i++) cache1.put(i, i); @@ -1305,12 +1347,16 @@ private void testCacheConfig(OutputFormat outputFormat) throws Exception { String outStr = testOut.toString(); if (outputFormat == null || outputFormat == SINGLE_LINE) { - assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME)); + for (int i = 0; i < cachesCnt; i++) + assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME + i)); + assertTrue(outStr.contains("partitions=32")); assertTrue(outStr.contains("function=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction")); } else if (outputFormat == MULTI_LINE) { - assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + "']")); + for (int i = 0; i < cachesCnt; i++) + assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + i + "']")); + assertTrue(outStr.contains("Affinity Partitions: 32")); assertTrue(outStr.contains("Affinity Function: o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction")); } From 51839c7178b4b426213fabcd99fab34de52234b2 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Wed, 7 Nov 2018 14:25:48 +0300 Subject: [PATCH 084/403] IGNITE-9975 Fixed LOST partitions handling - Fixes #5214. Signed-off-by: Alexey Goncharuk (cherry picked from commit b9cec42) --- .../GridDhtPartitionsExchangeFuture.java | 10 +- .../dht/preloader/GridDhtPreloader.java | 3 + .../GridDhtPartitionTopologyImpl.java | 9 +- .../ignite/cache/ResetLostPartitionTest.java | 260 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite5.java | 1 - .../testsuites/IgnitePdsTestSuite4.java | 3 + 6 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 5387180bef09c..616cfb45683db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -549,6 +549,14 @@ private boolean dynamicCacheStartExchange() { && exchActions.cacheStopRequests().isEmpty(); } + /** + * @param cacheOrGroupName Group or cache name for reset lost partitions. + * @return {@code True} if reset lost partition exchange. + */ + public boolean resetLostPartitionFor(String cacheOrGroupName) { + return exchActions != null && exchActions.cachesToResetLostPartitions().contains(cacheOrGroupName); + } + /** * @return {@code True} if activate cluster exchange. */ @@ -2177,8 +2185,6 @@ else if (log.isInfoEnabled()) { } } - exchActions = null; - if (firstDiscoEvt instanceof DiscoveryCustomEvent) ((DiscoveryCustomEvent)firstDiscoEvt).customMessage(null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java index 034bf72cecf4e..fa0c2b7b55976 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java @@ -168,6 +168,9 @@ private IgniteCheckedException stopError() { if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE)) return false; // No-op. + if (exchFut.resetLostPartitionFor(grp.cacheOrGroupName())) + return true; + if (exchFut.localJoinExchange()) return true; // Required, can have outdated updSeq partition counter if node reconnects. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 5f5155cd7626c..2e555f44bc77d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -2123,8 +2123,15 @@ else if (plc != PartitionLossPolicy.IGNORE) { if (locPart != null && locPart.state() == LOST) { boolean marked = locPart.own(); - if (marked) + if (marked) { updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer); + + long updateCntr = locPart.updateCounter(); + + //Set update counters to 0, for full rebalance. + locPart.updateCounter(updateCntr, -updateCntr); + locPart.initialUpdateCounter(0); + } } } } diff --git a/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java b/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java new file mode 100644 index 0000000000000..6f1e78bc23f8f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/cache/ResetLostPartitionTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cache; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; + +/** + * + */ +public class ResetLostPartitionTest extends GridCommonAbstractTest { + /** Ip finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** Cache name. */ + private static final String[] CACHE_NAMES = {"cacheOne", "cacheTwo", "cacheThree"}; + /** Cache size */ + public static final int CACHE_SIZE = 100000 / CACHE_NAMES.length; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setConsistentId(igniteInstanceName); + + DataStorageConfiguration storageCfg = new DataStorageConfiguration(); + + storageCfg.getDefaultDataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(300L * 1024 * 1024); + + cfg.setDataStorageConfiguration(storageCfg); + + CacheConfiguration[] ccfg = new CacheConfiguration[] { + cacheConfiguration(CACHE_NAMES[0], CacheAtomicityMode.ATOMIC), + cacheConfiguration(CACHE_NAMES[1], CacheAtomicityMode.ATOMIC), + cacheConfiguration(CACHE_NAMES[2], CacheAtomicityMode.TRANSACTIONAL) + }; + + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** + * @param cacheName Cache name. + * @param mode Cache atomicity mode. + * @return Configured cache configuration. + */ + private CacheConfiguration cacheConfiguration(String cacheName, CacheAtomicityMode mode) { + return new CacheConfiguration<>(cacheName) + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(mode) + .setBackups(1) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE) + .setAffinity(new RendezvousAffinityFunction(false, 1024)) + .setIndexedTypes(String.class, String.class); + } + + /** Client configuration */ + private IgniteConfiguration getClientConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setPeerClassLoadingEnabled(true); + cfg.setClientMode(true); + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + return cfg; + } + + /** + * Test to restore lost partitions after grid reactivation. + * + * @throws Exception if fail. + */ + public void testReactivateGridBeforeResetLostPartitions() throws Exception { + doRebalanceAfterPartitionsWereLost(true); + } + + /** + * Test to restore lost partitions on working grid. + * + * @throws Exception if fail. + */ + public void testResetLostPartitions() throws Exception { + doRebalanceAfterPartitionsWereLost(false); + } + + /** + * @param reactivateGridBeforeResetPart Reactive grid before try to reset lost partitions. + * @throws Exception if fail. + */ + private void doRebalanceAfterPartitionsWereLost(boolean reactivateGridBeforeResetPart) throws Exception { + startGrids(3); + + grid(0).cluster().active(true); + + Ignite igniteClient = startGrid(getClientConfiguration("client")); + + for (String cacheName : CACHE_NAMES) { + IgniteCache cache = igniteClient.cache(cacheName); + + for (int j = 0; j < CACHE_SIZE; j++) + cache.put(j, "Value" + j); + } + + stopGrid("client"); + + String dn2DirName = grid(1).name().replace(".", "_"); + + stopGrid(1); + + //Clean up the pds and WAL for second data node. + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR + "/" + dn2DirName, true)); + + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR + "/wal/" + dn2DirName, true)); + + //Here we have two from three data nodes and cache with 1 backup. So there is no data loss expected. + assertEquals(CACHE_NAMES.length * CACHE_SIZE, averageSizeAroundAllNodes()); + + //Start node 2 with empty PDS. Rebalance will be started. + startGrid(1); + + //During rebalance stop node 3. Rebalance will be stopped. + stopGrid(2); + + //Start node 3. + startGrid(2); + + //Loss data expected because rebalance to node 1 have not finished and node 2 was stopped. + assertTrue(CACHE_NAMES.length * CACHE_SIZE > averageSizeAroundAllNodes()); + + for (String cacheName : CACHE_NAMES) { + //Node 1 will have only OWNING partitions. + assertTrue(getPartitionsStates(0, cacheName).stream().allMatch(state -> state == OWNING)); + + //Node 3 will have only OWNING partitions. + assertTrue(getPartitionsStates(2, cacheName).stream().allMatch(state -> state == OWNING)); + } + + boolean hasLost = false; + for (String cacheName : CACHE_NAMES) { + //Node 2 will have OWNING and LOST partitions. + hasLost |= getPartitionsStates(1, cacheName).stream().anyMatch(state -> state == LOST); + } + + assertTrue(hasLost); + + if (reactivateGridBeforeResetPart) { + grid(0).cluster().active(false); + grid(0).cluster().active(true); + } + + //Try to reset lost partitions. + grid(2).resetLostPartitions(Arrays.asList(CACHE_NAMES)); + + awaitPartitionMapExchange(); + + for (String cacheName : CACHE_NAMES) { + //Node 2 will have only OWNING partitions. + assertTrue(getPartitionsStates(1, cacheName).stream().allMatch(state -> state == OWNING)); + } + + //All data was back. + assertEquals(CACHE_NAMES.length * CACHE_SIZE, averageSizeAroundAllNodes()); + + //Stop node 2 for checking rebalance correctness from this node. + stopGrid(2); + + //Rebalance should be successfully finished. + assertEquals(CACHE_NAMES.length * CACHE_SIZE, averageSizeAroundAllNodes()); + } + + /** + * @param gridNumber Grid number. + * @param cacheName Cache name. + * @return Partitions states for given cache name. + */ + private List getPartitionsStates(int gridNumber, String cacheName) { + CacheGroupContext cgCtx = grid(gridNumber).context().cache().cacheGroup(CU.cacheId(cacheName)); + + GridDhtPartitionTopologyImpl top = (GridDhtPartitionTopologyImpl)cgCtx.topology(); + + return top.localPartitions().stream() + .map(GridDhtLocalPartition::state) + .collect(Collectors.toList()); + } + + /** + * Checks that all nodes see the correct size. + */ + private int averageSizeAroundAllNodes() { + int totalSize = 0; + + for (Ignite ignite : IgnitionEx.allGrids()) { + for (String cacheName : CACHE_NAMES) { + totalSize += ignite.cache(cacheName).size(); + } + } + + return totalSize / IgnitionEx.allGrids().size(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index 109391da98c05..8776901ebf75f 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -103,7 +103,6 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(ConcurrentCacheStartTest.class); - //suite.addTestSuite(GridCacheAtomicPreloadSelfTest.class); //suite.addTestSuite(IgniteCacheContainsKeyColocatedAtomicSelfTest.class); //suite.addTestSuite(IgniteCacheContainsKeyNearAtomicSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java index 64615e20e0633..027f34105b7fd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.cache.ResetLostPartitionTest; import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; @@ -47,6 +48,8 @@ public static TestSuite suite() { suite.addTestSuite(IgnitePdsPartitionPreloadTest.class); + suite.addTestSuite(ResetLostPartitionTest.class); + return suite; } From 6d736b4680479808d0dc47d378b3001c0a74ba75 Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Fri, 9 Nov 2018 20:53:10 +0300 Subject: [PATCH 085/403] IGNITE-9843 Fixed invalid metadata handling on client reconnect - Fixes #5027. Signed-off-by: Alexey Goncharuk --- .../CacheObjectBinaryProcessorImpl.java | 43 +++++++++++-------- .../processors/query/h2/IgniteH2Indexing.java | 12 +++++- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index 137db9f887dc5..f92e104d82bcd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -33,6 +33,7 @@ import javax.cache.CacheException; import org.apache.ignite.IgniteBinary; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteClientDisconnectedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; @@ -46,9 +47,9 @@ import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.events.Event; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteNodeAttributes; import org.apache.ignite.internal.UnregisteredBinaryTypeException; import org.apache.ignite.internal.binary.BinaryContext; @@ -66,7 +67,6 @@ import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; import org.apache.ignite.internal.binary.streams.BinaryInputStream; import org.apache.ignite.internal.binary.streams.BinaryOffheapInputStream; -import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; import org.apache.ignite.internal.processors.cache.CacheObjectValueContext; @@ -105,7 +105,6 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_TEST_FEATURES_ENABLED; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAIT_SCHEMA_UPDATE; import static org.apache.ignite.IgniteSystemProperties.getBoolean; -import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED; import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.BINARY_PROC; /** @@ -116,6 +115,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm /** */ private volatile boolean discoveryStarted; + /** */ + private volatile IgniteFuture reconnectFut; + /** */ private BinaryContext binaryCtx; @@ -145,16 +147,6 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm @GridToStringExclude private IgniteBinary binaries; - /** Listener removes all registered binary schemas and user type descriptors after the local client reconnected. */ - private final GridLocalEventListener clientDisconLsnr = new GridLocalEventListener() { - @Override public void onEvent(Event evt) { - binaryContext().unregisterUserTypeDescriptors(); - binaryContext().unregisterBinarySchemas(); - - metadataLocCache.clear(); - } - }; - /** Locally cached metadata. This local cache is managed by exchanging discovery custom events. */ private final ConcurrentMap metadataLocCache = new ConcurrentHashMap<>(); @@ -176,9 +168,6 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void start() throws IgniteCheckedException { if (marsh instanceof BinaryMarshaller) { - if (ctx.clientNode()) - ctx.event().addLocalEventListener(clientDisconLsnr, EVT_CLIENT_NODE_DISCONNECTED); - if (!ctx.clientNode()) metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log, binaryMetadataFileStoreDir); @@ -285,17 +274,28 @@ public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener lsnr) /** {@inheritDoc} */ @Override public void stop(boolean cancel) { - if (ctx.clientNode()) - ctx.event().removeLocalEventListener(clientDisconLsnr); - if (transport != null) transport.stop(); } /** {@inheritDoc} */ @Override public void onDisconnected(IgniteFuture reconnectFut) throws IgniteCheckedException { + this.reconnectFut = reconnectFut; + if (transport != null) transport.onDisconnected(); + + binaryContext().unregisterUserTypeDescriptors(); + binaryContext().unregisterBinarySchemas(); + + metadataLocCache.clear(); + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture onReconnected(boolean clusterRestarted) throws IgniteCheckedException { + this.reconnectFut = null; + + return super.onReconnected(clusterRestarted); } /** {@inheritDoc} */ @@ -622,6 +622,11 @@ public GridBinaryMarshaller marshaller() { holder = metadataLocCache.get(typeId); + IgniteFuture reconnectFut0 = reconnectFut; + + if (holder == null && reconnectFut0 != null) + throw new IgniteClientDisconnectedException(reconnectFut0, "Client node disconnected."); + if (log.isDebugEnabled()) log.debug("Finished waiting for client metadata update" + " [typeId=" + typeId diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 820792f1d22a4..d3486aed79ee7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1887,7 +1887,17 @@ UpdateResult runDistributedUpdate( } @Override public Cache.Entry next() { - List l = iter0.next(); + List l; + + try { + l = iter0.next(); + } + catch (CacheException e) { + throw e; + } + catch (Exception e) { + throw new CacheException(e); + } return new CacheEntryImpl<>((K)l.get(0), (V)l.get(1)); } From 9b03078a29dc56533c0aa96ec9de6b5ddd919666 Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Thu, 8 Nov 2018 12:56:22 +0300 Subject: [PATCH 086/403] IGNITE-9840 Fixed deadlock on transactional future on client node caused by pending objects dump - Fixes #5268. Signed-off-by: Alexey Goncharuk (cherry picked from commit e1601f2) --- .../internal/binary/BinaryObjectExImpl.java | 6 +++ .../builder/BinaryObjectBuilderImpl.java | 3 +- .../CacheObjectBinaryProcessorImpl.java | 6 ++- .../apache/ignite/thread/IgniteThread.java | 44 ++++++++++++------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java index 920a296856213..f213ad916cc51 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.thread.IgniteThread; import org.jetbrains.annotations.Nullable; /** @@ -201,12 +202,17 @@ private String toString(BinaryReaderHandles ctx, IdentityHashMap 0) { diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java index 6f65e0e34d39b..e37e6a749bc7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java +++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java @@ -57,10 +57,10 @@ public class IgniteThread extends Thread { private final byte plc; /** */ - private boolean executingEntryProcessor; + private boolean holdsTopLock; /** */ - private boolean holdsTopLock; + private boolean forbiddenToRequestBinaryMetadata; /** * Creates thread with given worker. @@ -164,17 +164,10 @@ public void compositeRwLockIndex(int compositeRwLockIdx) { } /** - * @return {@code True} if thread is currently executing entry processor. - */ - public boolean executingEntryProcessor() { - return executingEntryProcessor; - } - - /** - * @return {@code True} if thread is currently holds topology lock. + * @return {@code True} if thread is not allowed to request binary metadata to avoid potential deadlock. */ - public boolean holdsTopLock() { - return holdsTopLock; + public boolean isForbiddenToRequestBinaryMetadata() { + return holdsTopLock || forbiddenToRequestBinaryMetadata; } /** @@ -183,11 +176,8 @@ public boolean holdsTopLock() { public static void onEntryProcessorEntered(boolean holdsTopLock) { Thread curThread = Thread.currentThread(); - if (curThread instanceof IgniteThread) { - ((IgniteThread)curThread).executingEntryProcessor = true; - + if (curThread instanceof IgniteThread) ((IgniteThread)curThread).holdsTopLock = holdsTopLock; - } } /** @@ -197,7 +187,27 @@ public static void onEntryProcessorLeft() { Thread curThread = Thread.currentThread(); if (curThread instanceof IgniteThread) - ((IgniteThread)curThread).executingEntryProcessor = false; + ((IgniteThread)curThread).holdsTopLock = false; + } + + /** + * Callback on entering critical section where binary metadata requests are forbidden. + */ + public static void onForbidBinaryMetadataRequestSectionEntered() { + Thread curThread = Thread.currentThread(); + + if (curThread instanceof IgniteThread) + ((IgniteThread)curThread).forbiddenToRequestBinaryMetadata = true; + } + + /** + * Callback on leaving critical section where binary metadata requests are forbidden. + */ + public static void onForbidBinaryMetadataRequestSectionLeft() { + Thread curThread = Thread.currentThread(); + + if (curThread instanceof IgniteThread) + ((IgniteThread)curThread).forbiddenToRequestBinaryMetadata = false; } /** From 86be297ddf3b7e4be31366fa535b481b2b483eea Mon Sep 17 00:00:00 2001 From: Max-Pudov Date: Tue, 13 Nov 2018 20:19:37 +0300 Subject: [PATCH 087/403] IGNITE-7793: Made SQL work what a value object has a field which name equals to affinity key name. This closes #4770. (cherry picked from commit dcdb27a24a450f63487290360b265e1570534629) --- .../internal/processors/query/QueryUtils.java | 15 +- .../query/property/QueryBinaryProperty.java | 8 +- .../query/h2/sql/GridSqlQuerySplitter.java | 25 ++ ...yKeyNameAndValueFieldNameConflictTest.java | 261 ++++++++++++++++++ .../IgniteCacheQuerySelfTestSuite.java | 3 + 5 files changed, 297 insertions(+), 15 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/AffinityKeyNameAndValueFieldNameConflictTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java index d977a49006648..1e0ab295c6540 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java @@ -825,17 +825,10 @@ public static QueryBinaryProperty buildBinaryProperty(GridKernalContext ctx, Str public static QueryClassProperty buildClassProperty(Class keyCls, Class valCls, String pathStr, Class resType, Map aliases, boolean notNull, CacheObjectContext coCtx) throws IgniteCheckedException { - QueryClassProperty res = buildClassProperty( - true, - keyCls, - pathStr, - resType, - aliases, - notNull, - coCtx); - - if (res == null) // We check key before value consistently with BinaryProperty. - res = buildClassProperty(false, valCls, pathStr, resType, aliases, notNull, coCtx); + QueryClassProperty res = buildClassProperty(false, valCls, pathStr, resType, aliases, notNull, coCtx); + + if (res == null) // We check value before key consistently with BinaryProperty. + res = buildClassProperty(true, keyCls, pathStr, resType, aliases, notNull, coCtx); if (res == null) throw new IgniteCheckedException(propertyInitializationExceptionMessage(keyCls, valCls, pathStr, resType)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java index 7a47c2fb3f1a3..3f2d233a75cfe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryBinaryProperty.java @@ -131,11 +131,11 @@ public QueryBinaryProperty(GridKernalContext ctx, String propName, QueryBinaryPr if (isKeyProp0 == 0) { // Key is allowed to be a non-binary object here. - // We check key before value consistently with ClassProperty. - if (key instanceof BinaryObject && ((BinaryObject)key).hasField(propName)) - isKeyProp = isKeyProp0 = 1; - else if (val instanceof BinaryObject && ((BinaryObject)val).hasField(propName)) + // We check value before key consistently with ClassProperty. + if (val instanceof BinaryObject && ((BinaryObject)val).hasField(propName)) isKeyProp = isKeyProp0 = -1; + else if (key instanceof BinaryObject && ((BinaryObject)key).hasField(propName)) + isKeyProp = isKeyProp0 = 1; else { if (!warned) { U.warn(log, "Neither key nor value have property \"" + propName + "\" " + diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index ca9c5bb064cb3..53b8e62af3b05 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -58,6 +58,7 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2CollocationModel.isCollocated; +import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT; import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst.TRUE; import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.AVG; import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.CAST; @@ -2375,6 +2376,9 @@ private static CacheQueryPartitionInfo extractPartitionFromEquality(GridSqlOpera GridH2Table tbl = (GridH2Table) column.column().getTable(); + if (!isAffinityKey(leftCol.getColumnId(), tbl)) + return null; + GridH2RowDescriptor desc = tbl.rowDescriptor(); IndexColumn affKeyCol = tbl.getAffinityKeyColumn(); @@ -2397,6 +2401,27 @@ private static CacheQueryPartitionInfo extractPartitionFromEquality(GridSqlOpera column.column().getType(), param.index()); } + /** + * + * @param colId Column ID to check + * @param tbl H2 Table + * @return is affinity key or not + */ + private static boolean isAffinityKey(int colId, GridH2Table tbl) { + GridH2RowDescriptor desc = tbl.rowDescriptor(); + + if (desc.isKeyColumn(colId)) + return true; + + IndexColumn affKeyCol = tbl.getAffinityKeyColumn(); + + try { + return affKeyCol != null && colId >= DEFAULT_COLUMNS_COUNT && desc.isColumnKeyProperty(colId - DEFAULT_COLUMNS_COUNT) && colId == affKeyCol.column.getColumnId(); + } catch(IllegalStateException e) { + return false; + } + } + /** * Merges two partition info arrays, removing duplicates * diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/AffinityKeyNameAndValueFieldNameConflictTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/AffinityKeyNameAndValueFieldNameConflictTest.java new file mode 100644 index 0000000000000..67db94c69d97e --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/AffinityKeyNameAndValueFieldNameConflictTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.Serializable; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheKeyConfiguration; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * IGNITE-7793 SQL does not work if value has sql field which name equals to affinity keyProducer name + */ +public class AffinityKeyNameAndValueFieldNameConflictTest extends GridCommonAbstractTest { + /** */ + private static final String PERSON_CACHE = "person"; + + /** */ + private Class keyCls; + + /** */ + private BiFunction keyProducer; + + /** */ + private boolean qryEntityCfg; + + /** */ + private boolean keyFieldSpecified; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = new CacheConfiguration(PERSON_CACHE); + + if (qryEntityCfg) { + CacheKeyConfiguration keyCfg = new CacheKeyConfiguration(keyCls.getName(), "name"); + cfg.setCacheKeyConfiguration(keyCfg); + + QueryEntity entity = new QueryEntity(); + entity.setKeyType(keyCls.getName()); + entity.setValueType(Person.class.getName()); + if (keyFieldSpecified) + entity.setKeyFields(Stream.of("name").collect(Collectors.toSet())); + + entity.addQueryField("id", Integer.class.getName(), null); + entity.addQueryField("name", String.class.getName(), null); + + ccfg.setQueryEntities(F.asList(entity)); + } else { + CacheKeyConfiguration keyCfg = new CacheKeyConfiguration(keyCls); + cfg.setCacheKeyConfiguration(keyCfg); + + ccfg.setIndexedTypes(keyCls, Person.class); + } + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** + * @throws Exception If failed. + */ + public void testQueryEntityConfig() throws Exception { + qryEntityCfg = true; + keyCls = PersonKey1.class; + keyProducer = PersonKey1::new; + checkQuery(); + } + + /** + * @throws Exception If failed. + */ + public void testQueryEntityConfigKeySpecified() throws Exception { + qryEntityCfg = true; + keyFieldSpecified = true; + keyCls = PersonKey1.class; + keyProducer = PersonKey1::new; + checkQuery(); + } + + /** + * @throws Exception If failed. + */ + public void testAnnotationConfig() throws Exception { + keyCls = PersonKey1.class; + keyProducer = PersonKey1::new; + checkQuery(); + } + + /** + * @throws Exception If failed. + */ + public void testAnnotationConfigCollision() throws Exception { + keyCls = PersonKey2.class; + keyProducer = PersonKey2::new; + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + checkQuery(); + + return null; + } + }, CacheException.class, "Property with name 'name' already exists."); + } + + /** + * @throws Exception If failed. + */ + private void checkQuery() throws Exception { + startGrid(2); + + Ignite g = grid(2); + + IgniteCache personCache = g.cache(PERSON_CACHE); + + personCache.put(keyProducer.apply(1, "o1"), new Person("p1")); + + SqlFieldsQuery query = new SqlFieldsQuery("select * from \"" + PERSON_CACHE + "\"." + Person.class.getSimpleName() + " it where it.name=?"); + + List> result = personCache.query(query.setArgs(keyFieldSpecified ? "o1" : "p1")).getAll(); + + assertEquals(1, result.size()); + + stopAllGrids(); + } + + /** + * + */ + public static class PersonKey1 { + /** */ + @QuerySqlField + private int id; + + /** */ + @AffinityKeyMapped + private String name; + + /** + * @param id Key. + * @param name Affinity keyProducer. + */ + public PersonKey1(int id, String name) { + this.id = id; + this.name = name; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + PersonKey1 other = (PersonKey1)o; + + return id == other.id; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return id; + } + } + + /** + * + */ + public static class PersonKey2 { + /** */ + @QuerySqlField + private int id; + + /** */ + @QuerySqlField + @AffinityKeyMapped + private String name; + + /** + * @param id Key. + * @param name Affinity keyProducer. + */ + public PersonKey2(int id, String name) { + this.id = id; + this.name = name; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + PersonKey2 other = (PersonKey2)o; + + return id == other.id; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return id; + } + } + + /** + * + */ + private static class Person implements Serializable { + + /** */ + @QuerySqlField + String name; + + /** + * @param name name. + */ + public Person(String name) { + this.name = name; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(Person.class, this); + } + } +} \ No newline at end of file diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index 801d51e2754dc..cd6dfc0a29c9c 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.AffinityKeyNameAndValueFieldNameConflictTest; import org.apache.ignite.internal.processors.cache.CacheIteratorScanQueryTest; import org.apache.ignite.internal.processors.cache.CacheLocalQueryDetailMetricsSelfTest; import org.apache.ignite.internal.processors.cache.CacheLocalQueryMetricsSelfTest; @@ -214,6 +215,8 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite { public static TestSuite suite() throws Exception { IgniteTestSuite suite = new IgniteTestSuite("Ignite Cache Queries Test Suite"); + suite.addTestSuite(AffinityKeyNameAndValueFieldNameConflictTest.class); + suite.addTestSuite(PartitionedSqlTest.class); suite.addTestSuite(ReplicatedSqlTest.class); From 4da5640ccfcf4569a86a8e97270e5053488938d2 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Wed, 14 Nov 2018 15:12:56 +0300 Subject: [PATCH 088/403] IGNITE-10235 Do not register cache in QueryManager twice if parallel caches start is disabled - Fixes #5381. Signed-off-by: Alexey Goncharuk (cherry picked from commit 00b9e89e7add9c58ed9a996aaba9200267c71961) --- .../processors/cache/GridCacheProcessor.java | 7 +- .../cache/StartCachesInParallelTest.java | 148 ++++++++++++++++++ ...heWithIndexingAndPersistenceTestSuite.java | 2 + 3 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 2e8dcf050226c..ed08806ade6d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2204,12 +2204,13 @@ public void prepareCacheStart( ) throws IgniteCheckedException { GridCacheContext cacheCtx = prepareCacheContext(startCfg, desc, reqNearCfg, exchTopVer, disabledAfterStart); - ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); - if (cacheCtx.isRecoveryMode()) finishRecovery(exchTopVer, cacheCtx); - else + else { + ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); + onCacheStarted(cacheCtx); + } } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java new file mode 100644 index 0000000000000..e48c294a5127b --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.Ignite; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.StopNodeFailureHandler; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL; + +/** + * Tests, that cluster could start and activate with all possible values of IGNITE_ALLOW_START_CACHES_IN_PARALLEL. + */ +public class StartCachesInParallelTest extends GridCommonAbstractTest { + /** IGNITE_ALLOW_START_CACHES_IN_PARALLEL option value before tests. */ + private String allowParallel; + + /** Test failure handler. */ + private TestStopNodeFailureHandler failureHnd; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))); + + cfg.setCacheConfiguration( + new CacheConfiguration<>() + .setName(DEFAULT_CACHE_NAME) + .setIndexedTypes(Integer.class, Integer.class)); + + failureHnd = new TestStopNodeFailureHandler(); + + cfg.setFailureHandler(failureHnd); + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + allowParallel = System.getProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + if (allowParallel != null) + System.setProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL, allowParallel); + else + System.clearProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** */ + public void testWithEnabledOption() throws Exception { + doTest("true"); + } + + /** */ + public void testWithDisabledOption() throws Exception { + doTest("false"); + } + + /** */ + public void testWithoutOption() throws Exception { + doTest(null); + } + + /** + * Test routine. + * + * @param optionVal IGNITE_ALLOW_START_CACHES_IN_PARALLEL value. + * @throws Exception If failed. + */ + private void doTest(String optionVal) throws Exception { + if (optionVal == null) + System.clearProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL); + else { + Boolean.parseBoolean(optionVal); + + System.setProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL, optionVal); + } + + assertEquals("Property wasn't set", optionVal, System.getProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL)); + + IgniteEx node = startGrid(0); + + node.cluster().active(true); + + assertNull("Node failed with " + failureHnd.lastFailureCtx, failureHnd.lastFailureCtx); + + assertTrue(node.cluster().active()); + } + + /** */ + private static class TestStopNodeFailureHandler extends StopNodeFailureHandler { + /** Last failure context. */ + private volatile FailureContext lastFailureCtx; + + /** {@inheritDoc} */ + @Override public boolean handle(Ignite ignite, FailureContext failureCtx) { + lastFailureCtx = failureCtx; + + return super.handle(ignite, failureCtx); + } + } +} \ No newline at end of file diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java index 1bc60f65658c8..2f3ddecef0d66 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.StartCachesInParallelTest; import org.apache.ignite.util.GridCommandHandlerIndexingTest; /** @@ -32,6 +33,7 @@ public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("Ignite Cache With Indexing And Persistence Test Suite"); suite.addTestSuite(GridCommandHandlerIndexingTest.class); + suite.addTestSuite(StartCachesInParallelTest.class); return suite; } From c820348155a35bffdd4bc47642c7646fa49ab539 Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Wed, 14 Nov 2018 15:40:57 +0300 Subject: [PATCH 089/403] IGNITE-10228 Fixed concurrent start caches race leading to a missed cache context - Fixes #5377. Signed-off-by: Alexey Goncharuk --- .../processors/cache/CacheGroupContext.java | 99 ++++----- .../ignite/internal/util/IgniteUtils.java | 39 ++-- .../distributed/CacheParallelStartTest.java | 202 ++++++++++++++++++ .../internal/util/IgniteUtilsSelfTest.java | 61 +++--- .../testsuites/IgniteCacheTestSuite6.java | 3 + 5 files changed, 305 insertions(+), 99 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index b0d6c07acc27a..93ba69c25960d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -20,7 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Iterator; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -115,10 +115,9 @@ public class CacheGroupContext { /** */ private final boolean storeCacheId; - /** */ - private volatile List caches; + /** We modify content under lock, by making defencive copy, field always contains unmodifiable list. */ + private volatile List caches = Collections.unmodifiableList(new ArrayList<>()); - /** */ private volatile List contQryCaches; /** */ @@ -152,16 +151,16 @@ public class CacheGroupContext { private final ReuseList reuseList; /** */ - private boolean drEnabled; + private volatile boolean drEnabled; /** */ - private boolean qryEnabled; + private volatile boolean qryEnabled; /** */ - private boolean mvccEnabled; + private final boolean mvccEnabled; /** MXBean. */ - private CacheGroupMetricsMXBean mxBean; + private final CacheGroupMetricsMXBean mxBean; /** */ private volatile boolean localWalEnabled; @@ -236,8 +235,6 @@ public class CacheGroupContext { log = ctx.kernalContext().log(getClass()); - caches = new ArrayList<>(); - mxBean = new CacheGroupMetricsMXBeanImpl(this); } @@ -307,10 +304,9 @@ void onCacheStarted(GridCacheContext cctx) throws IgniteCheckedException { public boolean hasCache(String cacheName) { List caches = this.caches; - for (int i = 0; i < caches.size(); i++) { - if (caches.get(i).name().equals(cacheName)) + for (GridCacheContext cacheContext : caches) + if (cacheContext.name().equals(cacheName)) return true; - } return false; } @@ -322,11 +318,17 @@ private void addCacheContext(GridCacheContext cctx) { assert cacheType.userCache() == cctx.userCache() : cctx.name(); assert grpId == cctx.groupId() : cctx.name(); - ArrayList caches = new ArrayList<>(this.caches); + final boolean add; - assert sharedGroup() || caches.isEmpty(); + synchronized (this) { + List copy = new ArrayList<>(caches); + + assert sharedGroup() || copy.isEmpty(); - boolean add = caches.add(cctx); + add = copy.add(cctx); + + caches = Collections.unmodifiableList(copy); + } assert add : cctx.name(); @@ -335,39 +337,39 @@ private void addCacheContext(GridCacheContext cctx) { if (!drEnabled && cctx.isDrEnabled()) drEnabled = true; - - this.caches = caches; } /** * @param cctx Cache context. */ private void removeCacheContext(GridCacheContext cctx) { - ArrayList caches = new ArrayList<>(this.caches); + final List copy; - // It is possible cache was not added in case of errors on cache start. - for (Iterator it = caches.iterator(); it.hasNext();) { - GridCacheContext next = it.next(); + synchronized (this) { + copy = new ArrayList<>(caches); - if (next == cctx) { - assert sharedGroup() || caches.size() == 1 : caches.size(); + for (GridCacheContext next : copy) { + if (next == cctx) { + assert sharedGroup() || copy.size() == 1 : copy.size(); - it.remove(); + copy.remove(next); - break; + break; + } } + + caches = Collections.unmodifiableList(copy); } if (QueryUtils.isEnabled(cctx.config())) { boolean qryEnabled = false; - for (int i = 0; i < caches.size(); i++) { - if (QueryUtils.isEnabled(caches.get(i).config())) { + for (GridCacheContext cacheContext : copy) + if (QueryUtils.isEnabled(cacheContext.config())) { qryEnabled = true; break; } - } this.qryEnabled = qryEnabled; } @@ -375,18 +377,15 @@ private void removeCacheContext(GridCacheContext cctx) { if (cctx.isDrEnabled()) { boolean drEnabled = false; - for (int i = 0; i < caches.size(); i++) { - if (caches.get(i).isDrEnabled()) { + for (GridCacheContext cacheContext : copy) + if (QueryUtils.isEnabled(cacheContext.config())) { drEnabled = true; break; } - } this.drEnabled = drEnabled; } - - this.caches = caches; } /** @@ -408,11 +407,8 @@ public GridCacheContext singleCacheContext() { public void unwindUndeploys() { List caches = this.caches; - for (int i = 0; i < caches.size(); i++) { - GridCacheContext cctx = caches.get(i); - + for (GridCacheContext cctx : caches) cctx.deploy().unwind(cctx); - } } /** @@ -450,9 +446,7 @@ public void addRebalanceEvent(int part, int type, ClusterNode discoNode, int dis List caches = this.caches; - for (int i = 0; i < caches.size(); i++) { - GridCacheContext cctx = caches.get(i); - + for (GridCacheContext cctx : caches) if (!cctx.config().isEventsDisabled() && cctx.recordEvent(type)) { cctx.gridEvents().record(new CacheRebalancingEvent(cctx.name(), cctx.localNode(), @@ -463,7 +457,6 @@ public void addRebalanceEvent(int part, int type, ClusterNode discoNode, int dis discoType, discoTs)); } - } } /** @@ -478,9 +471,7 @@ public void addUnloadEvent(int part) { List caches = this.caches; - for (int i = 0; i < caches.size(); i++) { - GridCacheContext cctx = caches.get(i); - + for (GridCacheContext cctx : caches) if (!cctx.config().isEventsDisabled()) cctx.gridEvents().record(new CacheRebalancingEvent(cctx.name(), cctx.localNode(), @@ -490,7 +481,6 @@ public void addUnloadEvent(int part) { null, 0, 0)); - } } /** @@ -517,9 +507,7 @@ public void addCacheEvent( ) { List caches = this.caches; - for (int i = 0; i < caches.size(); i++) { - GridCacheContext cctx = caches.get(i); - + for (GridCacheContext cctx : caches) if (!cctx.config().isEventsDisabled()) cctx.events().addEvent(part, key, @@ -535,7 +523,6 @@ public void addCacheEvent( null, null, keepBinary); - } } /** @@ -976,17 +963,19 @@ public Set cacheIds() { Set ids = U.newHashSet(caches.size()); - for (int i = 0; i < caches.size(); i++) - ids.add(caches.get(i).cacheId()); + for (GridCacheContext cctx : caches) + ids.add(cctx.cacheId()); return ids; } /** * @return Caches in this group. + * + * caches is already Unmodifiable list, so we don't need to explicitly wrap it here. */ public List caches() { - return this.caches; + return caches; } /** @@ -1004,9 +993,7 @@ public boolean hasCaches() { public void onPartitionEvicted(int part) { List caches = this.caches; - for (int i = 0; i < caches.size(); i++) { - GridCacheContext cctx = caches.get(i); - + for (GridCacheContext cctx : caches) { if (cctx.isDrEnabled()) cctx.dr().partitionEvicted(part); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index dc72b37a964e6..a06a35d2e1f0e 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -249,7 +249,6 @@ import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -10797,26 +10796,19 @@ public static Collection doInParallel( ) throws IgniteCheckedException, IgniteInterruptedCheckedException { if(srcDatas.isEmpty()) return Collections.emptyList(); + int[] batchSizes = calculateOptimalBatchSizes(parallelismLvl, srcDatas.size()); - int batchSize = srcDatas.size() / parallelismLvl; + List> batches = new ArrayList<>(batchSizes.length); - final int finalBatchSize = batchSize == 0 ? srcDatas.size() : batchSize; + Iterator iterator = srcDatas.iterator(); - List> batches = IntStream.range(0, parallelismLvl) - .mapToObj(i -> new ArrayList(finalBatchSize)) - .collect(Collectors.toList()); - - int batchIndex = 0; - - final int maxBatchIndex = batches.size() -1; + for (int batchSize : batchSizes) { + List batch = new ArrayList<>(batchSize); - List currentBatch = batches.get(batchIndex); + for (int i = 0; i < batchSize; i++) + batch.add(iterator.next()); - for (T src : srcDatas) { - currentBatch.add(src); - - if(currentBatch.size() >= batchSize && batchIndex < maxBatchIndex) - currentBatch = batches.get(++batchIndex); + batches.add(batch); } List>> consumerFutures = batches.stream() @@ -10874,6 +10866,21 @@ public static Collection doInParallel( return results; } + /** + * Split number of tasks into optimized batches. + * @param parallelismLvl Level of parallelism. + * @param size number of tasks to split. + * @return array of batch sizes. + */ + public static int[] calculateOptimalBatchSizes(int parallelismLvl, int size) { + int[] batcheSizes = new int[Math.min(parallelismLvl, size)]; + + for (int i = 0; i < size; i++) + batcheSizes[i % batcheSizes.length]++; + + return batcheSizes; + } + /** * @param fut Future to wait for completion. * @throws ExecutionException If the future diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java new file mode 100644 index 0000000000000..2a644856ca09c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.ArrayList; +import java.util.Collection; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; + +/** + * Test covers parallel start and stop of caches. + */ +public class CacheParallelStartTest extends GridCommonAbstractTest { + /** */ + private static final int CACHES_COUNT = 500; + + /** */ + private static final String STATIC_CACHE_PREFIX = "static-cache-"; + + /** */ + private static final String STATIC_CACHE_CACHE_GROUP_NAME = "static-cache-group"; + + /** */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** + * {@inheritDoc} + */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setSystemThreadPoolSize(Runtime.getRuntime().availableProcessors() * 3); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + long sz = 100 * 1024 * 1024; + + DataStorageConfiguration memCfg = new DataStorageConfiguration().setPageSize(1024) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setPersistenceEnabled(false).setInitialSize(sz).setMaxSize(sz)) + .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(24L * 60 * 60 * 1000); + + cfg.setDataStorageConfiguration(memCfg); + + ArrayList staticCaches = new ArrayList<>(CACHES_COUNT); + + for (int i = 0; i < CACHES_COUNT; i++) + staticCaches.add(cacheConfiguration(STATIC_CACHE_PREFIX + i)); + + cfg.setCacheConfiguration(staticCaches.toArray(new CacheConfiguration[CACHES_COUNT])); + + return cfg; + } + + /** + * @param cacheName Cache name. + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration(String cacheName) { + CacheConfiguration cfg = defaultCacheConfiguration(); + + cfg.setName(cacheName); + cfg.setBackups(1); + cfg.setGroupName(STATIC_CACHE_CACHE_GROUP_NAME); + cfg.setIndexedTypes(Long.class, Long.class); + + return cfg; + } + + /** + * {@inheritDoc} + */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanupTestData(); + } + + /** + * {@inheritDoc} + */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanupTestData(); + } + + /** */ + private void cleanupTestData() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.clearProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL); + } + + /** + * @throws Exception If failed. + */ + public void testParallelStartAndStop() throws Exception { + testParallelStartAndStop(true); + } + + /** + * @throws Exception if failed. + */ + public void testSequentialStartAndStop() throws Exception { + testParallelStartAndStop(false); + } + + /** + * + */ + private void testParallelStartAndStop(boolean parallel) throws Exception { + System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, String.valueOf(parallel)); + + IgniteEx igniteEx = startGrid(0); + + IgniteEx igniteEx2 = startGrid(1); + + igniteEx.cluster().active(true); + + assertCaches(igniteEx); + + assertCaches(igniteEx2); + + igniteEx.cluster().active(false); + + assertCachesAfterStop(igniteEx); + + assertCachesAfterStop(igniteEx2); + } + + /** + * + */ + private void assertCachesAfterStop(IgniteEx igniteEx) { + assertNull(igniteEx + .context() + .cache() + .cacheGroup(CU.cacheId(STATIC_CACHE_CACHE_GROUP_NAME))); + + assertTrue(igniteEx.context().cache().cacheGroups().isEmpty()); + + for (int i = 0; i < CACHES_COUNT; i++) { + assertNull(igniteEx.context().cache().cache(STATIC_CACHE_PREFIX + i)); + assertNull(igniteEx.context().cache().internalCache(STATIC_CACHE_PREFIX + i)); + } + } + + /** + * + */ + private void assertCaches(IgniteEx igniteEx) { + Collection caches = igniteEx + .context() + .cache() + .cacheGroup(CU.cacheId(STATIC_CACHE_CACHE_GROUP_NAME)) + .caches(); + + assertEquals(CACHES_COUNT, caches.size()); + + @Nullable CacheGroupContext cacheGroup = igniteEx + .context() + .cache() + .cacheGroup(CU.cacheId(STATIC_CACHE_CACHE_GROUP_NAME)); + + for (GridCacheContext cacheContext : caches) + assertEquals(cacheContext.group(), cacheGroup); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java index 4d417de6f06d7..eea11b6a8be43 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java @@ -943,6 +943,37 @@ public void testDoInParallelBatch() { } } + /** + * Test optimal splitting on batch sizes. + */ + public void testOptimalBatchSize() { + assertArrayEquals(new int[]{1}, IgniteUtils.calculateOptimalBatchSizes(1, 1)); + + assertArrayEquals(new int[]{2}, IgniteUtils.calculateOptimalBatchSizes(1, 2)); + + assertArrayEquals(new int[]{1, 1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(6, 4)); + + assertArrayEquals(new int[]{1}, IgniteUtils.calculateOptimalBatchSizes(4, 1)); + + assertArrayEquals(new int[]{1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 2)); + + assertArrayEquals(new int[]{1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 3)); + + assertArrayEquals(new int[]{1, 1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 4)); + + assertArrayEquals(new int[]{2, 1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 5)); + + assertArrayEquals(new int[]{2, 2, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 6)); + + assertArrayEquals(new int[]{2, 2, 2, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 7)); + + assertArrayEquals(new int[]{2, 2, 2, 2}, IgniteUtils.calculateOptimalBatchSizes(4, 8)); + + assertArrayEquals(new int[]{3, 2, 2, 2}, IgniteUtils.calculateOptimalBatchSizes(4, 9)); + + assertArrayEquals(new int[]{3, 3, 2, 2}, IgniteUtils.calculateOptimalBatchSizes(4, 10)); + } + /** * Test parallel execution in order. */ @@ -950,33 +981,9 @@ public void testDoInParallelResultsOrder() throws IgniteCheckedException { ExecutorService executorService = Executors.newFixedThreadPool(4); try { - testOrder(executorService, 1, 1); - testOrder(executorService, 2, 1); - testOrder(executorService, 3, 1); - testOrder(executorService, 9, 1); - testOrder(executorService, 10, 1); - testOrder(executorService, 9999, 1); - - testOrder(executorService, 1, 2); - testOrder(executorService, 2, 2); - testOrder(executorService, 3, 2); - testOrder(executorService, 9, 2); - testOrder(executorService, 10, 2); - testOrder(executorService, 9999, 2); - - testOrder(executorService, 1, 3); - testOrder(executorService, 2, 3); - testOrder(executorService, 3, 3); - testOrder(executorService, 3, 3); - testOrder(executorService, 10, 3); - testOrder(executorService, 9999, 3); - - testOrder(executorService, 1, 4); - testOrder(executorService, 2, 4); - testOrder(executorService, 3, 4); - testOrder(executorService, 9, 4); - testOrder(executorService, 10, 4); - testOrder(executorService, 9999, 4); + for(int parallelism = 1; parallelism < 16; parallelism++) + for(int size = 0; size < 10_000; size++) + testOrder(executorService, size, parallelism); } finally { executorService.shutdownNow(); } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index 18ae7f8e8791e..5300d38ae555c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest; import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest; import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheParallelStartTest; import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest; import org.apache.ignite.internal.processors.cache.distributed.CacheTryLockMultithreadedTest; import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest; @@ -110,6 +111,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class); suite.addTestSuite(CacheTryLockMultithreadedTest.class); + suite.addTestSuite(CacheParallelStartTest.class); + //suite.addTestSuite(CacheClientsConcurrentStartTest.class); //suite.addTestSuite(GridCacheRebalancingOrderingTest.class); //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class); From 02456e8bfc11c8410be8bb1356478a0879655a8a Mon Sep 17 00:00:00 2001 From: Max-Pudov Date: Wed, 14 Nov 2018 17:26:26 +0300 Subject: [PATCH 090/403] IGNITE-7793: compilation fix --- .../internal/processors/query/h2/sql/GridSqlQuerySplitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 53b8e62af3b05..51bd84242f0a3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -2376,7 +2376,7 @@ private static CacheQueryPartitionInfo extractPartitionFromEquality(GridSqlOpera GridH2Table tbl = (GridH2Table) column.column().getTable(); - if (!isAffinityKey(leftCol.getColumnId(), tbl)) + if (!isAffinityKey(column.column().getColumnId(), tbl)) return null; GridH2RowDescriptor desc = tbl.rowDescriptor(); From e85f9199da828059076acf58f85b5cdfc1cfa8d5 Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Thu, 15 Nov 2018 15:28:22 +0300 Subject: [PATCH 091/403] IGNITE-8735 Metastorage creates its own index partition - Fixes #5383. Signed-off-by: Ivan Rakov (cherry picked from commit a74d9c4f66f407bfde56173a727b2545cc3bfb99) --- .../internal/pagemem/PageIdAllocator.java | 3 + .../persistence/metastorage/MetaStorage.java | 7 ++ .../metastorage/MetastorageTree.java | 9 +++ .../IgniteMetaStorageBasicTest.java | 78 +++++++++++++++++++ 4 files changed, 97 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java index c6aeabe087975..b3e4b07b9fb67 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java @@ -35,6 +35,9 @@ public interface PageIdAllocator { /** Special partition reserved for index space. */ public static final int INDEX_PARTITION = 0xFFFF; + /** Special partition reserved for metastore space. */ + public static final int METASTORE_PARTITION = 0x0; + /** * Allocates a page from the space for the given partition ID and the given flags. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java index 4a243aaea005a..91fbec096029d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java @@ -28,6 +28,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -62,6 +63,7 @@ import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.jetbrains.annotations.NotNull; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId; import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId; @@ -507,6 +509,11 @@ public static class FreeListImpl extends AbstractFreeList { return SimpleDataPageIO.VERSIONS; } + /** {@inheritDoc} */ + @Override protected long allocatePageNoReuse() throws IgniteCheckedException { + return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA); + } + /** * Read row from data pages. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java index 00db5cd791163..27de6841c9084 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java @@ -19,6 +19,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -31,6 +32,9 @@ import org.apache.ignite.internal.processors.failure.FailureProcessor; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; + /** * */ @@ -94,6 +98,11 @@ public MetastorageRowStore rowStore() { return rowStore; } + /** {@inheritDoc} */ + @Override protected long allocatePageNoReuse() throws IgniteCheckedException { + return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java index e5a53fac1833b..b6dfc12826177 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java @@ -17,6 +17,9 @@ package org.apache.ignite.internal.processors.cache.persistence.metastorage; import java.io.Serializable; +import java.util.HashSet; +import java.util.List; +import java.util.Random; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -78,6 +81,81 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest { cleanPersistenceDir(); } + /** + * + */ + public void testMetaStorageMassivePutFixed() throws Exception { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database(); + + MetaStorage metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + Random rnd = new Random(); + + db.checkpointReadLock(); + + int size; + try { + for (int i = 0; i < 10_000; i++) { + size = rnd.nextBoolean() ? 3500 : 2 * 3500; + String key = "TEST_KEY_" + (i % 1000); + + byte[] arr = new byte[size]; + rnd.nextBytes(arr); + + metaStorage.remove(key); + + metaStorage.putData(key, arr/*b.toString().getBytes()*/); + } + } + finally { + db.checkpointReadUnlock(); + } + } + + /** + * + */ + public void testMetaStorageMassivePutRandom() throws Exception { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database(); + + MetaStorage metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + Random rnd = new Random(); + + db.checkpointReadLock(); + + int size; + try { + for (int i = 0; i < 50_000; i++) { + size = 100 + rnd.nextInt(9000); + + String key = "TEST_KEY_" + (i % 2_000); + + byte[] arr = new byte[size]; + rnd.nextBytes(arr); + + metaStorage.remove(key); + + metaStorage.putData(key, arr/*b.toString().getBytes()*/); + } + } + finally { + db.checkpointReadUnlock(); + } + } + /** * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully * after restart. From 111e33a29ee2ad50a6efeab59f34705637b702c1 Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Thu, 1 Nov 2018 19:41:38 +0300 Subject: [PATCH 092/403] IGNITE-10045 Add fail-fast mode to bounded iteration of StandaloneWalRecordsIterator - Fixes #5199. Signed-off-by: Dmitriy Pavlov --- .../wal/reader/IgniteWalIteratorFactory.java | 16 ++- .../reader/StandaloneWalRecordsIterator.java | 56 ++++++++- .../StandaloneWalRecordsIteratorTest.java | 115 +++++++++++++++++- 3 files changed, 181 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java index 4ea1828e6c189..f4ebec525dc7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java @@ -181,7 +181,8 @@ public WALIterator iterator( iteratorParametersBuilder.lowBound, iteratorParametersBuilder.highBound, iteratorParametersBuilder.keepBinary, - iteratorParametersBuilder.bufferSize + iteratorParametersBuilder.bufferSize, + iteratorParametersBuilder.strictBoundsCheck ); } @@ -418,6 +419,9 @@ public static class IteratorParametersBuilder { /** */ private FileWALPointer highBound = DFLT_HIGH_BOUND; + /** Use strict bounds check for WAL segments. */ + private boolean strictBoundsCheck; + /** * @param filesOrDirs Paths to files or directories. * @return IteratorParametersBuilder Self reference. @@ -534,6 +538,16 @@ public IteratorParametersBuilder to(FileWALPointer highBound) { return this; } + /** + * @param flag Use strict check. + * @return IteratorParametersBuilder Self reference. + */ + public IteratorParametersBuilder strictBoundsCheck(boolean flag) { + this.strictBoundsCheck = flag; + + return this; + } + /** * Copy current state of builder to new instance. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index 25432d3d38973..be5f55a1bb4fb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.GridKernalContext; @@ -116,7 +117,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { FileWALPointer lowBound, FileWALPointer highBound, boolean keepBinary, - int initialReadBufferSize + int initialReadBufferSize, + boolean strictBoundsCheck ) throws IgniteCheckedException { super( log, @@ -127,6 +129,9 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { FILE_INPUT_FACTORY ); + if (strictBoundsCheck) + strictCheck(walFiles, lowBound, highBound); + this.lowBound = lowBound; this.highBound = highBound; @@ -139,6 +144,55 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { advance(); } + /** + * @param walFiles Wal files. + * @return printable indexes of segment files. + */ + private static String printIndexes(List walFiles) { + return "[" + String.join(",", walFiles.stream().map(f -> Long.toString(f.idx())).collect(Collectors.toList())) + "]"; + } + + /** + * @param walFiles Wal files. + * @param lowBound Low bound. + * @param highBound High bound. + * + * @throws IgniteCheckedException if failed + */ + private static void strictCheck(List walFiles, FileWALPointer lowBound, FileWALPointer highBound) throws IgniteCheckedException { + int idx = 0; + + if (lowBound.index() > Long.MIN_VALUE) { + for (; idx < walFiles.size(); idx++) { + FileDescriptor desc = walFiles.get(idx); + + assert desc != null; + + if (desc.idx() == lowBound.index()) + break; + } + } + + if (idx == walFiles.size()) + throw new IgniteCheckedException("Wal segments not in bounds. loBoundIndex=" + lowBound.index() + + ", indexes=" + printIndexes(walFiles)); + + long curWalSegmIdx = walFiles.get(idx).idx(); + + for (; idx < walFiles.size() && curWalSegmIdx <= highBound.index(); idx++, curWalSegmIdx++) { + FileDescriptor desc = walFiles.get(idx); + + assert desc != null; + + if (curWalSegmIdx != desc.idx()) + throw new IgniteCheckedException("Wal segment " + curWalSegmIdx + " not found in files " + printIndexes(walFiles)); + } + + if (highBound.index() < Long.MAX_VALUE && curWalSegmIdx <= highBound.index()) + throw new IgniteCheckedException("Wal segments not in bounds. hiBoundIndex=" + highBound.index() + + ", indexes=" + printIndexes(walFiles)); + } + /** * For directory mode sets oldest file as initial segment, for file by file mode, converts all files to descriptors * and gets oldest as initial. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java index 6aa2a33e77142..8132c126d6993 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; @@ -30,17 +32,23 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.RolloverType; import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder; @@ -89,11 +97,9 @@ public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest { } /** - * Check correct closing file descriptors. * - * @throws Exception if test failed. */ - public void testCorrectClosingFileDescriptors() throws Exception { + private String createWalFiles() throws Exception { IgniteEx ig = (IgniteEx)startGrid(); String archiveWalDir = getArchiveWalDirPath(ig); @@ -118,8 +124,18 @@ public void testCorrectClosingFileDescriptors() throws Exception { stopGrid(); + return archiveWalDir; + } + + /** + * Check correct closing file descriptors. + * + * @throws Exception if test failed. + */ + public void testCorrectClosingFileDescriptors() throws Exception { + // Iterate by all archived WAL segments. - createWalIterator(archiveWalDir).forEach(x -> { + createWalIterator(createWalFiles()).forEach(x -> { }); assertTrue("At least one WAL file must be opened!", CountedFileIO.getCountOpenedWalFiles() > 0); @@ -127,6 +143,62 @@ public void testCorrectClosingFileDescriptors() throws Exception { assertTrue("All WAL files must be closed at least ones!", CountedFileIO.getCountOpenedWalFiles() <= CountedFileIO.getCountClosedWalFiles()); } + /** + * Check correct check bounds. + * + * @throws Exception if test failed. + */ + public void testStrictBounds() throws Exception { + String dir = createWalFiles(); + + FileWALPointer lowBound = null, highBound = null; + + for (IgniteBiTuple p : createWalIterator(dir, null, null, false)) { + if (lowBound == null) + lowBound = (FileWALPointer) p.get1(); + + highBound = (FileWALPointer) p.get1(); + } + + assertNotNull(lowBound); + + assertNotNull(highBound); + + createWalIterator(dir, lowBound, highBound, true); + + final FileWALPointer lBound = lowBound; + final FileWALPointer hBound = highBound; + + //noinspection ThrowableNotThrown + GridTestUtils.assertThrows(log, () -> { + createWalIterator(dir, new FileWALPointer(lBound.index() - 1, 0, 0), hBound, true); + + return 0; + } , IgniteCheckedException.class, null); + + //noinspection ThrowableNotThrown + GridTestUtils.assertThrows(log, () -> { + createWalIterator(dir, lBound, new FileWALPointer(hBound.index() + 1, 0, 0), true); + + return 0; + }, IgniteCheckedException.class, null); + + List walFiles = listWalFiles(dir); + + assertNotNull(walFiles); + + assertTrue(!walFiles.isEmpty()); + + assertTrue(walFiles.get(new Random().nextInt(walFiles.size())).file().delete()); + + //noinspection ThrowableNotThrown + GridTestUtils.assertThrows(log, () -> { + createWalIterator(dir, lBound, hBound, true); + + return 0; + }, IgniteCheckedException.class, null); + } + /** * Creates WALIterator associated with files inside walDir. * @@ -142,6 +214,41 @@ private WALIterator createWalIterator(String walDir) throws IgniteCheckedExcepti return new IgniteWalIteratorFactory(log).iterator(params.filesOrDirs(walDir)); } + + /** + * @param walDir Wal directory. + */ + private List listWalFiles(String walDir) throws IgniteCheckedException { + IteratorParametersBuilder params = new IteratorParametersBuilder(); + + params.ioFactory(new RandomAccessFileIOFactory()); + + return new IgniteWalIteratorFactory(log).resolveWalFiles(params.filesOrDirs(walDir)); + } + + /** + * @param walDir Wal directory. + * @param lowBound Low bound. + * @param highBound High bound. + * @param strictCheck Strict check. + */ + private WALIterator createWalIterator(String walDir, FileWALPointer lowBound, FileWALPointer highBound, boolean strictCheck) + throws IgniteCheckedException { + IteratorParametersBuilder params = new IteratorParametersBuilder(); + + params.ioFactory(new RandomAccessFileIOFactory()). + filesOrDirs(walDir). + strictBoundsCheck(strictCheck); + + if (lowBound != null) + params.from(lowBound); + + if (lowBound != null) + params.to(highBound); + + return new IgniteWalIteratorFactory(log).iterator(params); + } + /** * Evaluate path to directory with WAL archive. * From ffe31d1fefdcf9bcb9edbfe18b8e39ce53877f0b Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Thu, 15 Nov 2018 16:33:07 +0300 Subject: [PATCH 093/403] IGNITE-10246 StandaloneWALRecordIterator must throw checkBounds exception - Fixes #5398. Signed-off-by: Ivan Rakov --- .../wal/reader/IgniteWalIteratorFactory.java | 3 ++- .../reader/StandaloneWalRecordsIterator.java | 6 +++--- .../wal/reader/StrictBoundsCheckException.java | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java index f4ebec525dc7c..697ddeda4d052 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java @@ -564,7 +564,8 @@ public IteratorParametersBuilder copy() { .marshallerMappingFileStoreDir(marshallerMappingFileStoreDir) .from(lowBound) .to(highBound) - .filter(filter); + .filter(filter) + .strictBoundsCheck(strictBoundsCheck); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index be5f55a1bb4fb..833d392a569bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -174,7 +174,7 @@ private static void strictCheck(List walFiles, FileWALPointer lo } if (idx == walFiles.size()) - throw new IgniteCheckedException("Wal segments not in bounds. loBoundIndex=" + lowBound.index() + + throw new StrictBoundsCheckException("Wal segments not in bounds. loBoundIndex=" + lowBound.index() + ", indexes=" + printIndexes(walFiles)); long curWalSegmIdx = walFiles.get(idx).idx(); @@ -185,11 +185,11 @@ private static void strictCheck(List walFiles, FileWALPointer lo assert desc != null; if (curWalSegmIdx != desc.idx()) - throw new IgniteCheckedException("Wal segment " + curWalSegmIdx + " not found in files " + printIndexes(walFiles)); + throw new StrictBoundsCheckException("Wal segment " + curWalSegmIdx + " not found in files " + printIndexes(walFiles)); } if (highBound.index() < Long.MAX_VALUE && curWalSegmIdx <= highBound.index()) - throw new IgniteCheckedException("Wal segments not in bounds. hiBoundIndex=" + highBound.index() + + throw new StrictBoundsCheckException("Wal segments not in bounds. hiBoundIndex=" + highBound.index() + ", indexes=" + printIndexes(walFiles)); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java new file mode 100644 index 0000000000000..4a52e0811cf64 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java @@ -0,0 +1,18 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal.reader; + +import org.apache.ignite.IgniteCheckedException; + +/** + * This exception is used in checking boundaries (StandaloneWalRecordsIterator). + */ +public class StrictBoundsCheckException extends IgniteCheckedException { + /** */ + private static final long serialVersionUID = 0L; + + /** + * @param mesg Message. + */ + public StrictBoundsCheckException(String mesg) { + super(mesg); + } +} \ No newline at end of file From 87ca70a3201c1e76fc8e4f35aa1ce5919a79058d Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Thu, 15 Nov 2018 19:15:13 +0300 Subject: [PATCH 094/403] IGNITE-10207 Fixed missed loss policy checks - Fixes #5360. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit e4760980ab6077ee398965f551fdf8302820ae0e) Signed-off-by: Dmitriy Govorukhin --- .../processors/cache/GridCacheContext.java | 19 +- .../distributed/dht/GridDhtGetFuture.java | 40 +- .../dht/GridDhtGetSingleFuture.java | 41 +- .../dht/GridDhtTopologyFutureAdapter.java | 233 ++++--- .../dht/GridPartitionedGetFuture.java | 2 +- .../dht/GridPartitionedSingleGetFuture.java | 9 +- .../GridDhtPartitionsExchangeFuture.java | 4 +- .../distributed/near/GridNearGetFuture.java | 2 +- ...cheResultIsNotNullOnPartitionLossTest.java | 23 +- ...gniteCachePartitionLossPolicySelfTest.java | 652 ++++++------------ ...exingCachePartitionLossPolicySelfTest.java | 2 +- 11 files changed, 483 insertions(+), 544 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index ff430b6eeea91..664dc7eafee45 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -2242,9 +2242,14 @@ else if (type == EVT_CACHE_REBALANCE_STOPPED) { * * @param affNodes All affinity nodes. * @param canRemap Flag indicating that 'get' should be done on a locked topology version. + * @param partitionId Partition ID. * @return Affinity node to get key from or {@code null} if there is no suitable alive node. */ - @Nullable public ClusterNode selectAffinityNodeBalanced(List affNodes, boolean canRemap) { + @Nullable public ClusterNode selectAffinityNodeBalanced( + List affNodes, + int partitionId, + boolean canRemap + ) { if (!readLoadBalancingEnabled) { if (!canRemap) { for (ClusterNode node : affNodes) { @@ -2268,7 +2273,7 @@ else if (type == EVT_CACHE_REBALANCE_STOPPED) { ClusterNode n0 = null; for (ClusterNode node : affNodes) { - if (canRemap || discovery().alive(node)) { + if ((canRemap || discovery().alive(node) && isOwner(node, partitionId))) { if (locMacs.equals(node.attribute(ATTR_MACS))) return node; @@ -2282,6 +2287,16 @@ else if (type == EVT_CACHE_REBALANCE_STOPPED) { return n0; } + /** + * Check that node is owner for partition. + * @param node Cluster node. + * @param partitionId Partition ID. + * @return {@code} + */ + private boolean isOwner(ClusterNode node, int partitionId) { + return topology().partitionState(node.id(), partitionId) == OWNING; + } + /** * Prepare affinity field for builder (if possible). * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java index 24fd621c8878a..b60c8a06a8310 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.ReaderArguments; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.LostPolicyValidator; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; @@ -55,6 +56,11 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static java.util.Collections.singleton; +import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.OperationType.READ; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; + /** * */ @@ -179,6 +185,7 @@ public GridDhtGetFuture( * Initializes future. */ void init() { + // TODO get rid of force keys request https://issues.apache.org/jira/browse/IGNITE-10251 GridDhtFuture fut = cctx.group().preloader().request(cctx, keys.keySet(), topVer); if (fut != null) { @@ -203,14 +210,14 @@ void init() { return; } - map0(keys); + map0(keys, true); markInitialized(); } }); } else { - map0(keys); + map0(keys, false); markInitialized(); } @@ -251,7 +258,7 @@ public GridCacheVersion version() { /** * @param keys Keys to map. */ - private void map0(Map keys) { + private void map0(Map keys, boolean forceKeys) { Map mappedKeys = null; // Assign keys to primary nodes. @@ -259,7 +266,7 @@ private void map0(Map keys) { int part = cctx.affinity().partition(key.getKey()); if (retries == null || !retries.contains(part)) { - if (!map(key.getKey())) { + if (!map(key.getKey(), forceKeys)) { if (retries == null) retries = new HashSet<>(); @@ -303,7 +310,7 @@ else if (mappedKeys != null) * @param key Key. * @return {@code True} if mapped. */ - private boolean map(KeyCacheObject key) { + private boolean map(KeyCacheObject key, boolean forceKeys) { try { int keyPart = cctx.affinity().partition(key); @@ -314,14 +321,31 @@ private boolean map(KeyCacheObject key) { if (part == null) return false; + if (!forceKeys && part.state() == LOST && !recovery) { + Throwable error = LostPolicyValidator.validate(cctx, key, READ, singleton(part.id())); + + if (error != null) { + onDone(null, error); + + return false; + } + } + if (parts == null || !F.contains(parts, part.id())) { // By reserving, we make sure that partition won't be unloaded while processed. if (part.reserve()) { - parts = parts == null ? new int[1] : Arrays.copyOf(parts, parts.length + 1); + if (forceKeys || (part.state() == OWNING || part.state() == LOST)) { + parts = parts == null ? new int[1] : Arrays.copyOf(parts, parts.length + 1); - parts[parts.length - 1] = part.id(); + parts[parts.length - 1] = part.id(); - return true; + return true; + } + else { + part.release(); + + return false; + } } else return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java index d722397442999..ea806635bb123 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java @@ -35,6 +35,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.ReaderArguments; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.LostPolicyValidator; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; @@ -47,6 +48,11 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static java.util.Collections.singleton; +import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.OperationType.READ; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; + /** * */ @@ -201,6 +207,7 @@ public GridCacheVersion version() { * */ private void map() { + // TODO get rid of force keys request https://issues.apache.org/jira/browse/IGNITE-10251 if (cctx.group().preloader().needForceKeys()) { GridDhtFuture fut = cctx.group().preloader().request( cctx, @@ -234,7 +241,7 @@ private void map() { onDone(e); } else - map0(); + map0(true); } } ); @@ -243,19 +250,20 @@ private void map() { } } - map0(); + map0(false); } /** * */ - private void map0() { + private void map0(boolean forceKeys) { assert retry == null : retry; - if (!map(key)) { + if (!map(key, forceKeys)) { retry = cctx.affinity().partition(key); - onDone((GridCacheEntryInfo)null); + if (!isDone()) + onDone((GridCacheEntryInfo)null); return; } @@ -272,7 +280,7 @@ private void map0() { * @param key Key. * @return {@code True} if mapped. */ - private boolean map(KeyCacheObject key) { + private boolean map(KeyCacheObject key, boolean forceKeys) { try { int keyPart = cctx.affinity().partition(key); @@ -285,11 +293,28 @@ private boolean map(KeyCacheObject key) { assert this.part == -1; + if (!forceKeys && part.state() == LOST && !recovery) { + Throwable error = LostPolicyValidator.validate(cctx, key, READ, singleton(part.id())); + + if (error != null) { + onDone(null, error); + + return false; + } + } + // By reserving, we make sure that partition won't be unloaded while processed. if (part.reserve()) { - this.part = part.id(); + if (forceKeys || (part.state() == OWNING || part.state() == LOST)) { + this.part = part.id(); + + return true; + } + else { + part.release(); - return true; + return false; + } } else return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java index 92143080c2409..8a6a5ee0a5a89 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java @@ -29,12 +29,13 @@ import org.apache.ignite.internal.processors.cache.CacheInvalidStateException; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_ALL; import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE; -import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_ALL; import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE; +import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter.OperationType.WRITE; /** * @@ -42,7 +43,7 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter implements GridDhtTopologyFuture { /** Cache groups validation results. */ - protected volatile Map grpValidRes; + protected volatile Map grpValidRes = Collections.emptyMap(); /** Whether or not cluster is active. */ protected volatile boolean clusterIsActive = true; @@ -52,7 +53,7 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter topNodes) { + protected final CacheGroupValidation validateCacheGroup(CacheGroupContext grp, Collection topNodes) { Collection lostParts = grp.isLocal() ? Collections.emptyList() : grp.topology().lostPartitions(); @@ -65,11 +66,11 @@ protected final CacheValidation validateCacheGroup(CacheGroupContext grp, Collec valid = validator.validate(topNodes); } - return new CacheValidation(valid, lostParts); + return new CacheGroupValidation(valid, lostParts); } /** {@inheritDoc} */ - @Nullable @Override public final Throwable validateCache( + @Override public final @Nullable Throwable validateCache( GridCacheContext cctx, boolean recovery, boolean read, @@ -87,115 +88,181 @@ protected final CacheValidation validateCacheGroup(CacheGroupContext grp, Collec return new CacheInvalidStateException( "Failed to perform cache operation (cluster is not activated): " + cctx.name()); + OperationType opType = read ? OperationType.READ : WRITE; + CacheGroupContext grp = cctx.group(); - PartitionLossPolicy partLossPlc = grp.config().getPartitionLossPolicy(); + PartitionLossPolicy lossPlc = grp.config().getPartitionLossPolicy(); + + if (cctx.shared().readOnlyMode() && opType == WRITE) + return new IgniteCheckedException("Failed to perform cache operation (cluster is in read only mode)"); if (grp.needsRecovery() && !recovery) { - if (!read && (partLossPlc == READ_ONLY_SAFE || partLossPlc == READ_ONLY_ALL)) - return new IgniteCheckedException("Failed to write to cache (cache is moved to a read-only state): " + - cctx.name()); + if (opType == WRITE && (lossPlc == READ_ONLY_SAFE || lossPlc == READ_ONLY_ALL)) + return new IgniteCheckedException( + "Failed to write to cache (cache is moved to a read-only state): " + cctx.name()); } - if (cctx.shared().readOnlyMode() && !read) - return new IgniteCheckedException("Failed to perform cache operation (cluster is in read only mode)" ); + CacheGroupValidation validation = grpValidRes.get(grp.groupId()); - if (grp.needsRecovery() || grp.topologyValidator() != null) { - CacheValidation validation = grpValidRes.get(grp.groupId()); + if (validation == null) + return null; - if (validation == null) - return null; + if (opType == WRITE && !validation.isValid()) { + return new IgniteCheckedException("Failed to perform cache operation " + + "(cache topology is not valid): " + cctx.name()); + } - if (!validation.valid && !read) - return new IgniteCheckedException("Failed to perform cache operation " + - "(cache topology is not valid): " + cctx.name()); + if (recovery) + return null; - if (recovery || !grp.needsRecovery()) - return null; + if (validation.hasLostPartitions()) { + if (key != null) + return LostPolicyValidator.validate(cctx, key, opType, validation.lostPartitions()); - if (key != null) { - int p = cctx.affinity().partition(key); + if (keys != null) + return LostPolicyValidator.validate(cctx, keys, opType, validation.lostPartitions()); + } - CacheInvalidStateException ex = validatePartitionOperation(cctx.name(), read, key, p, - validation.lostParts, partLossPlc); + return null; + } - if (ex != null) - return ex; - } + /** + * Cache group validation result. + */ + protected static class CacheGroupValidation { + /** Topology validation result. */ + private final boolean valid; - if (keys != null) { - for (Object k : keys) { - int p = cctx.affinity().partition(k); + /** Lost partitions on this topology version. */ + private final Collection lostParts; - CacheInvalidStateException ex = validatePartitionOperation(cctx.name(), read, k, p, - validation.lostParts, partLossPlc); + /** + * @param valid Valid flag. + * @param lostParts Lost partitions. + */ + private CacheGroupValidation(boolean valid, Collection lostParts) { + this.valid = valid; + this.lostParts = lostParts; + } - if (ex != null) - return ex; - } - } + /** + * @return True if valid, False if invalide. + */ + public boolean isValid() { + return valid; } - return null; + /** + * @return True if lost partition is present, False if not. + */ + public boolean hasLostPartitions() { + return !F.isEmpty(lostParts); + } + + /** + * @return Lost patition ID collection. + */ + public Collection lostPartitions() { + return lostParts; + } } /** - * @param cacheName Cache name. - * @param read Read flag. - * @param key Key to check. - * @param part Partition this key belongs to. - * @param lostParts Collection of lost partitions. - * @param plc Partition loss policy. - * @return Invalid state exception if this operation is disallowed. + * */ - private CacheInvalidStateException validatePartitionOperation( - String cacheName, - boolean read, - Object key, - int part, - Collection lostParts, - PartitionLossPolicy plc - ) { - if (lostParts.contains(part)) { - if (!read) { - assert plc == READ_WRITE_ALL || plc == READ_WRITE_SAFE; + public enum OperationType { + /** + * Read operation. + */ + READ, + /** + * Write operation. + */ + WRITE + } + + /** + * Lost policy validator. + */ + public static class LostPolicyValidator { + /** + * + */ + public static Throwable validate( + GridCacheContext cctx, + Object key, + OperationType opType, + Collection lostParts + ) { + CacheGroupContext grp = cctx.group(); + + PartitionLossPolicy lostPlc = grp.config().getPartitionLossPolicy(); + + int partition = cctx.affinity().partition(key); + + return validate(cctx, key, partition, opType, lostPlc, lostParts); + } + + /** + * + */ + public static Throwable validate( + GridCacheContext cctx, + Collection keys, + OperationType opType, + Collection lostParts + ) { + CacheGroupContext grp = cctx.group(); - if (plc == READ_WRITE_SAFE) { + PartitionLossPolicy lostPlc = grp.config().getPartitionLossPolicy(); + + for (Object key : keys) { + int partition = cctx.affinity().partition(key); + + Throwable res = validate(cctx, key, partition, opType, lostPlc, lostParts); + + if (res != null) + return res; + } + + return null; + } + + /** + * + */ + private static Throwable validate( + GridCacheContext cctx, + Object key, + int partition, + OperationType opType, + PartitionLossPolicy lostPlc, + Collection lostParts + ) { + if (opType == WRITE) { + if (lostPlc == READ_ONLY_SAFE || lostPlc == READ_ONLY_ALL) { + return new IgniteCheckedException( + "Failed to write to cache (cache is moved to a read-only state): " + cctx.name() + ); + } + + if (lostParts.contains(partition) && lostPlc == READ_WRITE_SAFE) { return new CacheInvalidStateException("Failed to execute cache operation " + "(all partition owners have left the grid, partition data has been lost) [" + - "cacheName=" + cacheName + ", part=" + part + ", key=" + key + ']'); + "cacheName=" + cctx.name() + ", part=" + partition + ", key=" + key + ']'); } } - else { - // Read. - if (plc == READ_ONLY_SAFE || plc == READ_WRITE_SAFE) + + if (opType == OperationType.READ) { + if (lostParts.contains(partition) && (lostPlc == READ_ONLY_SAFE || lostPlc == READ_WRITE_SAFE)) return new CacheInvalidStateException("Failed to execute cache operation " + "(all partition owners have left the grid, partition data has been lost) [" + - "cacheName=" + cacheName + ", part=" + part + ", key=" + key + ']'); + "cacheName=" + cctx.name() + ", part=" + partition + ", key=" + key + ']' + ); } - } - - return null; - } - - /** - * Cache validation result. - */ - protected static class CacheValidation { - /** Topology validation result. */ - private boolean valid; - /** Lost partitions on this topology version. */ - private Collection lostParts; - - /** - * @param valid Valid flag. - * @param lostParts Lost partitions. - */ - private CacheValidation(boolean valid, Collection lostParts) { - this.valid = valid; - this.lostParts = lostParts; + return null; } } - } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java index e09a599246081..9e91d66805e5f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java @@ -475,7 +475,7 @@ private boolean map( } } - ClusterNode node = cctx.selectAffinityNodeBalanced(affNodes, canRemap); + ClusterNode node = cctx.selectAffinityNodeBalanced(affNodes, part, canRemap); if (node == null) { onDone(serverNotFoundError(topVer)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java index 4dc72b2a0dd6e..6ae625c4cb042 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java @@ -358,7 +358,7 @@ private void map(final AffinityTopologyVersion topVer) { } } - ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, canRemap); + ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, part, canRemap); if (affNode == null) { onDone(serverNotFoundError(topVer)); @@ -764,6 +764,13 @@ private ClusterTopologyServerNotFoundException serverNotFoundError(AffinityTopol private void remap(final AffinityTopologyVersion topVer) { cctx.closures().runLocalSafe(new Runnable() { @Override public void run() { + GridDhtTopologyFuture lastFut = cctx.shared().exchange().lastFinishedFuture(); + + Throwable error = lastFut.validateCache(cctx, recovery, true, key, null); + + if (error != null) + onDone(error); + map(topVer); } }); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 616cfb45683db..122d835dc1977 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2089,10 +2089,10 @@ public void finishMerged() { } } - if (serverNodeDiscoveryEvent()) + if (serverNodeDiscoveryEvent() || localJoinExchange()) detectLostPartitions(res); - Map m = U.newHashMap(cctx.cache().cacheGroups().size()); + Map m = U.newHashMap(cctx.cache().cacheGroups().size()); for (CacheGroupContext grp : cctx.cache().cacheGroups()) m.put(grp.groupId(), validateCacheGroup(grp, events().lastEvent().topologyNodes())); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java index d7800ed61c758..b954a0b82b9db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java @@ -494,7 +494,7 @@ private Map map( } } - ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, canRemap); + ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, part, canRemap); if (affNode == null) { onDone(serverNotFoundError(topVer)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java index ceafc9e61600d..0958f839a03cf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java @@ -17,15 +17,21 @@ package org.apache.ignite.internal.processors.cache.distributed; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteClientDisconnectedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.PartitionLossPolicy; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.EventType; @@ -48,13 +54,13 @@ public class CacheResultIsNotNullOnPartitionLossTest extends GridCommonAbstractT private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); /** Number of servers to be started. */ - private static final int SERVERS = 10; + private static final int SERVERS = 5; /** Index of node that is goning to be the only client node. */ private static final int CLIENT_IDX = SERVERS; /** Number of cache entries to insert into the test cache. */ - private static final int CACHE_ENTRIES_CNT = 10_000; + private static final int CACHE_ENTRIES_CNT = 60; /** True if {@link #getConfiguration(String)} is expected to configure client node on next invocations. */ private boolean isClient; @@ -75,6 +81,7 @@ public class CacheResultIsNotNullOnPartitionLossTest extends GridCommonAbstractT .setCacheMode(CacheMode.PARTITIONED) .setBackups(0) .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAffinity(new RendezvousAffinityFunction(false, 50)) .setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE) ); @@ -90,7 +97,12 @@ public class CacheResultIsNotNullOnPartitionLossTest extends GridCommonAbstractT cleanPersistenceDir(); - startGrids(SERVERS); + List list = IntStream.range(0, SERVERS).boxed().collect(Collectors.toList()); + + Collections.shuffle(list); + + for (Integer i : list) + startGrid(i); isClient = true; @@ -178,9 +190,9 @@ private void testCacheResultIsNotNull0(IgniteEx ignite) throws Exception { readerThreadStarted.await(1, TimeUnit.SECONDS); for (int i = 0; i < SERVERS - 1; i++) { - Thread.sleep(50L); - grid(i).close(); + + Thread.sleep(400L); } } finally { @@ -204,6 +216,7 @@ private void testCacheResultIsNotNull0(IgniteEx ignite) throws Exception { private boolean expectedThrowableClass(Throwable throwable) { return X.hasCause( throwable, + IgniteClientDisconnectedException.class, CacheInvalidStateException.class, ClusterTopologyCheckedException.class, IllegalStateException.class, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java index caf0829f0417d..f02563d0d47d3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java @@ -18,14 +18,11 @@ package org.apache.ignite.internal.processors.cache.distributed; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; @@ -39,8 +36,6 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.CacheRebalancingEvent; import org.apache.ignite.events.Event; @@ -48,7 +43,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.TestDelayingCommunicationSpi; import org.apache.ignite.internal.managers.communication.GridIoMessage; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; import org.apache.ignite.internal.util.typedef.F; @@ -58,18 +53,19 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import static java.util.Arrays.asList; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; +import static org.apache.ignite.internal.processors.cache.GridCacheUtils.cacheId; /** * */ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTest { /** */ - private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); /** */ private boolean client; @@ -78,43 +74,36 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe private PartitionLossPolicy partLossPlc; /** */ - protected static final String CACHE_NAME = "partitioned"; + private int backups; /** */ - private int backups = 0; + private final AtomicBoolean delayPartExchange = new AtomicBoolean(); /** */ - private final AtomicBoolean delayPartExchange = new AtomicBoolean(false); - - /** */ - private final TopologyChanger killSingleNode = new TopologyChanger(false, Collections.singletonList(3), Arrays.asList(0, 1, 2, 4), 0); - - /** */ - private boolean isPersistenceEnabled; + private final TopologyChanger killSingleNode = new TopologyChanger( + false, asList(3), asList(0, 1, 2, 4), 0 + ); /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); - ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder); + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); cfg.setCommunicationSpi(new TestDelayingCommunicationSpi() { + /** {@inheritDoc} */ @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) { - return delayPartExchange.get() && (msg instanceof GridDhtPartitionsFullMessage || msg instanceof GridDhtPartitionsAbstractMessage); + return delayPartExchange.get() && + (msg instanceof GridDhtPartitionsFullMessage || msg instanceof GridDhtPartitionsAbstractMessage); } - @Override protected int delayMillis() { - return 250; - } }); cfg.setClientMode(client); - cfg.setCacheConfiguration(cacheConfiguration()); + cfg.setConsistentId(gridName); - cfg.setDataStorageConfiguration(new DataStorageConfiguration().setDefaultDataRegionConfiguration( - new DataRegionConfiguration().setPersistenceEnabled(isPersistenceEnabled) - )); + cfg.setCacheConfiguration(cacheConfiguration()); return cfg; } @@ -123,7 +112,7 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe * @return Cache configuration. */ protected CacheConfiguration cacheConfiguration() { - CacheConfiguration cacheCfg = new CacheConfiguration<>(CACHE_NAME); + CacheConfiguration cacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); cacheCfg.setCacheMode(PARTITIONED); cacheCfg.setBackups(backups); @@ -135,44 +124,27 @@ protected CacheConfiguration cacheConfiguration() { } /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - delayPartExchange.set(false); - - partLossPlc = PartitionLossPolicy.IGNORE; - - backups = 0; - - isPersistenceEnabled = false; + @Override protected void afterTest() throws Exception { + stopAllGrids(); } /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopAllGrids(); + @Override protected void beforeTest() throws Exception { + super.beforeTest(); cleanPersistenceDir(); - super.afterTest(); - } - - /** - * @throws Exception if failed. - */ - public void testReadOnlySafe() throws Exception { - partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE; + delayPartExchange.set(false); - checkLostPartition(false, true, killSingleNode); + backups = 0; } /** * @throws Exception if failed. */ - public void testReadOnlySafeWithPersistence() throws Exception { + public void testReadOnlySafe() throws Exception { partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE; - isPersistenceEnabled = true; - checkLostPartition(false, true, killSingleNode); } @@ -185,19 +157,6 @@ public void testReadOnlyAll() throws Exception { checkLostPartition(false, false, killSingleNode); } - /** - * @throws Exception if failed. - */ - public void testReadOnlyAllWithPersistence() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10041"); - - partLossPlc = PartitionLossPolicy.READ_ONLY_ALL; - - isPersistenceEnabled = true; - - checkLostPartition(false, false, killSingleNode); - } - /** * @throws Exception if failed. */ @@ -207,17 +166,6 @@ public void testReadWriteSafe() throws Exception { checkLostPartition(true, true, killSingleNode); } - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeWithPersistence() throws Exception { - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, killSingleNode); - } - /** * @throws Exception if failed. */ @@ -227,37 +175,13 @@ public void testReadWriteAll() throws Exception { checkLostPartition(true, false, killSingleNode); } - /** - * @throws Exception if failed. - */ - public void testReadWriteAllWithPersistence() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10041"); - - partLossPlc = PartitionLossPolicy.READ_WRITE_ALL; - - isPersistenceEnabled = true; - - checkLostPartition(true, false, killSingleNode); - } - /** * @throws Exception if failed. */ public void testReadWriteSafeAfterKillTwoNodes() throws Exception { partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); - } - - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeAfterKillTwoNodesWithPersistence() throws Exception { - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); + checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 0)); } /** @@ -266,18 +190,7 @@ public void testReadWriteSafeAfterKillTwoNodesWithPersistence() throws Exception public void testReadWriteSafeAfterKillTwoNodesWithDelay() throws Exception { partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 20)); - } - - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeAfterKillTwoNodesWithDelayWithPersistence() throws Exception { - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, new TopologyChanger(false, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 20)); + checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 20)); } /** @@ -288,22 +201,7 @@ public void testReadWriteSafeWithBackupsAfterKillThreeNodes() throws Exception { backups = 1; - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2, 1), Arrays.asList(0, 4), 0)); - } - - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeWithBackupsAfterKillThreeNodesWithPersistence() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10043"); - - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - backups = 1; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2, 1), Arrays.asList(0, 4), 0)); + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2, 1), asList(0, 4), 0)); } /** @@ -312,18 +210,7 @@ public void testReadWriteSafeWithBackupsAfterKillThreeNodesWithPersistence() thr public void testReadWriteSafeAfterKillCrd() throws Exception { partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); - } - - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeAfterKillCrdWithPersistence() throws Exception { - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0)); } /** @@ -334,20 +221,7 @@ public void testReadWriteSafeWithBackups() throws Exception { backups = 1; - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); - } - - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeWithBackupsWithPersistence() throws Exception { - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - backups = 1; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 2), Arrays.asList(0, 1, 4), 0)); + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2), asList(0, 1, 4), 0)); } /** @@ -358,95 +232,26 @@ public void testReadWriteSafeWithBackupsAfterKillCrd() throws Exception { backups = 1; - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); - } - - /** - * @throws Exception if failed. - */ - public void testReadWriteSafeWithBackupsAfterKillCrdWithPersistence() throws Exception { - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; - - backups = 1; - - isPersistenceEnabled = true; - - checkLostPartition(true, true, new TopologyChanger(true, Arrays.asList(3, 0), Arrays.asList(1, 2, 4), 0)); - } - - /** - * @throws Exception if failed. - */ - public void testIgnore() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-5078"); - - partLossPlc = PartitionLossPolicy.IGNORE; - - checkIgnore(killSingleNode); + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0)); } /** + * @param topChanger topology changer. * @throws Exception if failed. */ - public void testIgnoreWithPersistence() throws Exception { + public void testIgnore(TopologyChanger topChanger) throws Exception { fail("https://issues.apache.org/jira/browse/IGNITE-5078"); - fail("https://issues.apache.org/jira/browse/IGNITE-10041"); - - partLossPlc = PartitionLossPolicy.IGNORE; - - isPersistenceEnabled = true; - - checkIgnore(killSingleNode); - } - - /** - * @throws Exception if failed. - */ - public void testIgnoreKillThreeNodes() throws Exception { - partLossPlc = PartitionLossPolicy.IGNORE; - - // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078. - // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed. - // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0); - TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Collections.singletonList(0), 0); - - checkIgnore(onlyCrdIsAlive); - } - - /** - * @throws Exception if failed. - */ - public void testIgnoreKillThreeNodesWithPersistence() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10041"); - - partLossPlc = PartitionLossPolicy.IGNORE; - - isPersistenceEnabled = true; - - // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078. - // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed. - // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0); - TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Collections.singletonList(0), 0); - - checkIgnore(onlyCrdIsAlive); - } - - /** - * @param topChanger topology changer. - * @throws Exception if failed. - */ - private void checkIgnore(TopologyChanger topChanger) throws Exception { topChanger.changeTopology(); for (Ignite ig : G.allGrids()) { - IgniteCache cache = ig.cache(CACHE_NAME); + IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); Collection lost = cache.lostPartitions(); assertTrue("[grid=" + ig.name() + ", lost=" + lost.toString() + ']', lost.isEmpty()); - int parts = ig.affinity(CACHE_NAME).partitions(); + int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions(); for (int i = 0; i < parts; i++) { cache.get(i); @@ -465,127 +270,109 @@ private void checkIgnore(TopologyChanger topChanger) throws Exception { private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger topChanger) throws Exception { assert partLossPlc != null; - List lostParts = topChanger.changeTopology(); - - // Wait for all grids (servers and client) have same topology version - // to make sure that all nodes received map with lost partition. - boolean success = GridTestUtils.waitForCondition(() -> { - AffinityTopologyVersion last = null; - for (Ignite ig : G.allGrids()) { - AffinityTopologyVersion ver = ((IgniteEx)ig).context().cache().context().exchange().readyAffinityVersion(); - - if (last != null && !last.equals(ver)) - return false; - - last = ver; - } - - return true; - }, 10000); - - assertTrue("Failed to wait for new topology", success); + int part = topChanger.changeTopology().get(0); for (Ignite ig : G.allGrids()) { info("Checking node: " + ig.cluster().localNode().id()); - IgniteCache cache = ig.cache(CACHE_NAME); + IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); - verifyLostPartitions(ig, lostParts); - - verifyCacheOps(canWrite, safe, ig); - - validateQuery(safe, ig); + verifyCacheOps(canWrite, safe, part, ig); - // TODO withPartitionRecover doesn't work with BLT - https://issues.apache.org/jira/browse/IGNITE-10041. - if (!isPersistenceEnabled) { - // Check we can read and write to lost partition in recovery mode. - IgniteCache recoverCache = cache.withPartitionRecover(); + // Check we can read and write to lost partition in recovery mode. + IgniteCache recoverCache = cache.withPartitionRecover(); - for (int lostPart : recoverCache.lostPartitions()) { - recoverCache.get(lostPart); - recoverCache.put(lostPart, lostPart); - } - - // Check that writing in recover mode does not clear partition state. - verifyLostPartitions(ig, lostParts); + for (int lostPart : recoverCache.lostPartitions()) { + recoverCache.get(lostPart); + recoverCache.put(lostPart, lostPart); + } - verifyCacheOps(canWrite, safe, ig); + // Check that writing in recover mode does not clear partition state. + verifyCacheOps(canWrite, safe, part, ig); - validateQuery(safe, ig); - } + // Validate queries. + validateQuery(safe, ig); } - // Bring all nodes back. - for (int i : topChanger.killNodes) { - IgniteEx grd = startGrid(i); + checkNewNode(true, canWrite, safe, part); + checkNewNode(false, canWrite, safe, part); - info("Newly started node: " + grd.cluster().localNode().id()); + // Check that partition state does not change after we start a new node. + IgniteEx grd = startGrid(3); - // Check that partition state does not change after we start each node. - // TODO With persistence enabled LOST partitions become OWNING after a node joins back - https://issues.apache.org/jira/browse/IGNITE-10044. - if (!isPersistenceEnabled) { - for (Ignite ig : G.allGrids()) { - verifyCacheOps(canWrite, safe, ig); + info("Newly started node: " + grd.cluster().localNode().id()); - // TODO Query effectively waits for rebalance due to https://issues.apache.org/jira/browse/IGNITE-10057 - // TODO and after resetLostPartition there is another OWNING copy in the cluster due to https://issues.apache.org/jira/browse/IGNITE-10058. - // TODO Uncomment after https://issues.apache.org/jira/browse/IGNITE-10058 is fixed. -// validateQuery(safe, ig); - } - } - } + for (Ignite ig : G.allGrids()) + verifyCacheOps(canWrite, safe, part, ig); - ignite(4).resetLostPartitions(Collections.singletonList(CACHE_NAME)); + ignite(4).resetLostPartitions(Collections.singletonList(DEFAULT_CACHE_NAME)); awaitPartitionMapExchange(true, true, null); for (Ignite ig : G.allGrids()) { - IgniteCache cache = ig.cache(CACHE_NAME); + IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); assertTrue(cache.lostPartitions().isEmpty()); - int parts = ig.affinity(CACHE_NAME).partitions(); + int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions(); for (int i = 0; i < parts; i++) { cache.get(i); cache.put(i, i); } - - for (int i = 0; i < parts; i++) { - checkQueryPasses(ig, false, i); - - if (shouldExecuteLocalQuery(ig, i)) - checkQueryPasses(ig, true, i); - - } - - checkQueryPasses(ig, false); } } /** - * @param node Node. - * @param lostParts Lost partition IDs. + * @param client Client flag. + * @param canWrite Can write flag. + * @param safe Safe flag. + * @param part List of lost partitions. + * @throws Exception If failed to start a new node. */ - private void verifyLostPartitions(Ignite node, List lostParts) { - IgniteCache cache = node.cache(CACHE_NAME); + private void checkNewNode( + boolean client, + boolean canWrite, + boolean safe, + int part + ) throws Exception { + this.client = client; + + try { + IgniteEx cl = startGrid("newNode"); + + CacheGroupContext grpCtx = cl.context().cache().cacheGroup(cacheId(DEFAULT_CACHE_NAME)); + + assertTrue(grpCtx.needsRecovery()); - Set actualSortedLostParts = new TreeSet<>(cache.lostPartitions()); - Set expSortedLostParts = new TreeSet<>(lostParts); + verifyCacheOps(canWrite, safe, part, cl); - assertEqualsCollections(expSortedLostParts, actualSortedLostParts); + validateQuery(safe, cl); + } + finally { + stopGrid("newNode", false); + + this.client = false; + } } /** + * * @param canWrite {@code True} if writes are allowed. * @param safe {@code True} if lost partition should trigger exception. + * @param part Lost partition ID. * @param ig Ignite instance. */ - private void verifyCacheOps(boolean canWrite, boolean safe, Ignite ig) { - IgniteCache cache = ig.cache(CACHE_NAME); + private void verifyCacheOps(boolean canWrite, boolean safe, int part, Ignite ig) { + IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); - int parts = ig.affinity(CACHE_NAME).partitions(); + Collection lost = cache.lostPartitions(); + + assertTrue("Failed to find expected lost partition [exp=" + part + ", lost=" + lost + ']', + lost.contains(part)); + + int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions(); // Check read. for (int i = 0; i < parts; i++) { @@ -632,8 +419,8 @@ private void verifyCacheOps(boolean canWrite, boolean safe, Ignite ig) { * @param nodes List of nodes to find partition. * @return List of partitions that aren't primary or backup for specified nodes. */ - private List noPrimaryOrBackupPartition(List nodes) { - Affinity aff = ignite(4).affinity(CACHE_NAME); + protected List noPrimaryOrBackupPartition(List nodes) { + Affinity aff = ignite(4).affinity(DEFAULT_CACHE_NAME); List parts = new ArrayList<>(); @@ -657,6 +444,127 @@ private List noPrimaryOrBackupPartition(List nodes) { return parts; } + /** */ + private class TopologyChanger { + /** Flag to delay partition exchange */ + private boolean delayExchange; + + /** List of nodes to kill */ + private List killNodes; + + /** List of nodes to be alive */ + private List aliveNodes; + + /** Delay between node stops */ + private long stopDelay; + + /** + * @param delayExchange Flag for delay partition exchange. + * @param killNodes List of nodes to kill. + * @param aliveNodes List of nodes to be alive. + * @param stopDelay Delay between stopping nodes. + */ + public TopologyChanger( + boolean delayExchange, + List killNodes, + List aliveNodes, + long stopDelay + ) { + this.delayExchange = delayExchange; + this.killNodes = killNodes; + this.aliveNodes = aliveNodes; + this.stopDelay = stopDelay; + } + + /** + * @return Lost partition ID. + * @throws Exception If failed. + */ + protected List changeTopology() throws Exception { + startGrids(4); + + Affinity aff = ignite(0).affinity(DEFAULT_CACHE_NAME); + + for (int i = 0; i < aff.partitions(); i++) + ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i); + + client = true; + + startGrid(4); + + client = false; + + for (int i = 0; i < 5; i++) + info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']'); + + awaitPartitionMapExchange(); + + final List parts = noPrimaryOrBackupPartition(aliveNodes); + + if (parts.isEmpty()) + throw new IllegalStateException("No partition on nodes: " + killNodes); + + final List> lostMap = new ArrayList<>(); + + for (int i : aliveNodes) { + HashMap semaphoreMap = new HashMap<>(); + + for (Integer part : parts) + semaphoreMap.put(part, new Semaphore(0)); + + lostMap.add(semaphoreMap); + + grid(i).events().localListen(new P1() { + @Override public boolean apply(Event evt) { + assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; + + CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt; + + if (F.eq(DEFAULT_CACHE_NAME, cacheEvt.cacheName())) { + if (semaphoreMap.containsKey(cacheEvt.partition())) + semaphoreMap.get(cacheEvt.partition()).release(); + } + + return true; + } + }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST); + } + + if (delayExchange) + delayPartExchange.set(true); + + ExecutorService executor = Executors.newFixedThreadPool(killNodes.size()); + + for (Integer node : killNodes) { + executor.submit(new Runnable() { + @Override public void run() { + grid(node).close(); + } + }); + + Thread.sleep(stopDelay); + } + + executor.shutdown(); + + delayPartExchange.set(false); + + Thread.sleep(5_000L); + + for (Map map : lostMap) { + for (Map.Entry entry : map.entrySet()) + assertTrue("Failed to wait for partition LOST event for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); + } + + for (Map map : lostMap) { + for (Map.Entry entry : map.entrySet()) + assertFalse("Partition LOST event raised twice for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); + } + + return parts; + } + } + /** * Validate query execution on a node. * @@ -665,7 +573,7 @@ private List noPrimaryOrBackupPartition(List nodes) { */ private void validateQuery(boolean safe, Ignite node) { // Get node lost and remaining partitions. - IgniteCache cache = node.cache(CACHE_NAME); + IgniteCache cache = node.cache(DEFAULT_CACHE_NAME); Collection lostParts = cache.lostPartitions(); @@ -673,7 +581,7 @@ private void validateQuery(boolean safe, Ignite node) { Integer remainingPart = null; - for (int i = 0; i < node.affinity(CACHE_NAME).partitions(); i++) { + for (int i = 0; i < node.affinity(DEFAULT_CACHE_NAME).partitions(); i++) { if (lostParts.contains(i)) continue; @@ -730,7 +638,7 @@ private boolean shouldExecuteLocalQuery(Ignite node, int... parts) { int numOfPrimaryParts = 0; - for (int nodePrimaryPart : node.affinity(CACHE_NAME).primaryPartitions(node.cluster().localNode())) { + for (int nodePrimaryPart : node.affinity(DEFAULT_CACHE_NAME).primaryPartitions(node.cluster().localNode())) { for (int part : parts) { if (part == nodePrimaryPart) numOfPrimaryParts++; @@ -754,7 +662,7 @@ protected void checkQueryPasses(Ignite node, boolean loc, int... parts) { if (loc) return; - IgniteCache cache = node.cache(CACHE_NAME); + IgniteCache cache = node.cache(DEFAULT_CACHE_NAME); ScanQuery qry = new ScanQuery(); @@ -777,124 +685,4 @@ protected void checkQueryFails(Ignite node, boolean loc, int... parts) { // TODO Need to add an actual check after https://issues.apache.org/jira/browse/IGNITE-9902 is fixed. // No-op. } - - /** */ - private class TopologyChanger { - /** Flag to delay partition exchange */ - private boolean delayExchange; - - /** List of nodes to kill */ - private List killNodes; - - /** List of nodes to be alive */ - private List aliveNodes; - - /** Delay between node stops */ - private long stopDelay; - - /** - * @param delayExchange Flag for delay partition exchange. - * @param killNodes List of nodes to kill. - * @param aliveNodes List of nodes to be alive. - * @param stopDelay Delay between stopping nodes. - */ - private TopologyChanger(boolean delayExchange, List killNodes, List aliveNodes, - long stopDelay) { - this.delayExchange = delayExchange; - this.killNodes = killNodes; - this.aliveNodes = aliveNodes; - this.stopDelay = stopDelay; - } - - /** - * @return Lost partition ID. - * @throws Exception If failed. - */ - private List changeTopology() throws Exception { - startGrids(4); - - if (isPersistenceEnabled) - grid(0).cluster().active(true); - - Affinity aff = ignite(0).affinity(CACHE_NAME); - - for (int i = 0; i < aff.partitions(); i++) - ignite(0).cache(CACHE_NAME).put(i, i); - - client = true; - - startGrid(4); - - client = false; - - for (int i = 0; i < 5; i++) - info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']'); - - awaitPartitionMapExchange(); - - final List parts = noPrimaryOrBackupPartition(aliveNodes); - - if (parts.isEmpty()) - throw new IllegalStateException("No partition on nodes: " + killNodes); - - final List> lostMap = new ArrayList<>(); - - for (int i : aliveNodes) { - HashMap semaphoreMap = new HashMap<>(); - - for (Integer part : parts) - semaphoreMap.put(part, new Semaphore(0)); - - lostMap.add(semaphoreMap); - - grid(i).events().localListen(new P1() { - @Override public boolean apply(Event evt) { - assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; - - CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt; - - if (F.eq(CACHE_NAME, cacheEvt.cacheName())) { - if (semaphoreMap.containsKey(cacheEvt.partition())) - semaphoreMap.get(cacheEvt.partition()).release(); - } - - return true; - } - }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST); - } - - if (delayExchange) - delayPartExchange.set(true); - - ExecutorService executor = Executors.newFixedThreadPool(killNodes.size()); - - for (Integer node : killNodes) { - executor.submit(new Runnable() { - @Override public void run() { - grid(node).close(); - } - }); - - Thread.sleep(stopDelay); - } - - executor.shutdown(); - - delayPartExchange.set(false); - - Thread.sleep(5_000L); - - for (Map map : lostMap) { - for (Map.Entry entry : map.entrySet()) - assertTrue("Failed to wait for partition LOST event for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); - } - - for (Map map : lostMap) { - for (Map.Entry entry : map.entrySet()) - assertFalse("Partition LOST event raised twice for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); - } - - return parts; - } - } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java index a31a1c6110d50..7007499a84552 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java @@ -69,7 +69,7 @@ protected void checkQueryFails(Ignite node, boolean loc, int... parts) { * @param loc Local flag. */ private static void executeQuery(Ignite node, boolean loc, int... parts) { - IgniteCache cache = node.cache(CACHE_NAME); + IgniteCache cache = node.cache(DEFAULT_CACHE_NAME); SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * FROM Integer"); From 5f7d72a6a6f8a90ac2724316c51bc9a979f250d3 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Thu, 15 Nov 2018 20:14:33 +0300 Subject: [PATCH 095/403] IGNITE-10246 Added license header. (cherry picked from commit a802074) --- .../wal/reader/StrictBoundsCheckException.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java index 4a52e0811cf64..89d54eb1a2e28 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StrictBoundsCheckException.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.wal.reader; import org.apache.ignite.IgniteCheckedException; From c6671be14abc40cb9a70456312a9b98c62c9e3dc Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 16 Nov 2018 16:17:01 +0700 Subject: [PATCH 096/403] IGNITE-10293 Collect caches info only on active cluster. (cherry picked from commit eed26bcfb26d58372459f84173c4d0969dc08c8a) --- .../ignite/internal/visor/node/VisorNodeDataCollectorJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index 5fab8d16d5e94..9025ed084b9c5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -335,7 +335,8 @@ protected VisorNodeDataCollectorJobResult run(VisorNodeDataCollectorJobResult re if (debug) start0 = log(ignite.log(), "Collected memory metrics", getClass(), start0); - caches(res, arg); + if (ignite.cluster().active()) + caches(res, arg); if (debug) start0 = log(ignite.log(), "Collected caches", getClass(), start0); From 8a17703ff582339f8dba9e96cd3c8849cda019da Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 16 Nov 2018 16:17:01 +0700 Subject: [PATCH 097/403] IGNITE-10293 Collect caches info only on active cluster. (cherry picked from commit eed26bcfb26d58372459f84173c4d0969dc08c8a) --- .../ignite/internal/visor/node/VisorNodeDataCollectorJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index 5fab8d16d5e94..9025ed084b9c5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -335,7 +335,8 @@ protected VisorNodeDataCollectorJobResult run(VisorNodeDataCollectorJobResult re if (debug) start0 = log(ignite.log(), "Collected memory metrics", getClass(), start0); - caches(res, arg); + if (ignite.cluster().active()) + caches(res, arg); if (debug) start0 = log(ignite.log(), "Collected caches", getClass(), start0); From 3f8bd9081c923698a9c08e3aeb9dfd43d1f7730b Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Thu, 15 Nov 2018 16:59:54 +0300 Subject: [PATCH 098/403] IGNITE-9897: ODBC fix for PDO when used with unixODBC This closes #5248 (cherry picked from commit 59db29138406b6986200910ef8de44666f993fcd) --- modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp | 2 +- modules/platforms/cpp/odbc/src/config/connection_info.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp b/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp index d8ed0879aec4d..96e6164e5f21b 100644 --- a/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp +++ b/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp @@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(TestValues) CheckIntInfo(SQL_POS_OPERATIONS, 0); CheckIntInfo(SQL_SQL92_DATETIME_FUNCTIONS, SQL_SDF_CURRENT_DATE | SQL_SDF_CURRENT_TIMESTAMP); CheckIntInfo(SQL_SQL92_VALUE_EXPRESSIONS, SQL_SVE_CASE | SQL_SVE_CAST | SQL_SVE_COALESCE | SQL_SVE_NULLIF); - CheckIntInfo(SQL_STATIC_CURSOR_ATTRIBUTES1, SQL_CA1_NEXT); + CheckIntInfo(SQL_STATIC_CURSOR_ATTRIBUTES1, SQL_CA1_NEXT | SQL_CA1_ABSOLUTE); CheckIntInfo(SQL_STATIC_CURSOR_ATTRIBUTES2, 0); CheckIntInfo(SQL_PARAM_ARRAY_ROW_COUNTS, SQL_PARC_BATCH); CheckIntInfo(SQL_PARAM_ARRAY_SELECTS, SQL_PAS_NO_SELECT); diff --git a/modules/platforms/cpp/odbc/src/config/connection_info.cpp b/modules/platforms/cpp/odbc/src/config/connection_info.cpp index 5885381b0532b..fde8ca53c710f 100644 --- a/modules/platforms/cpp/odbc/src/config/connection_info.cpp +++ b/modules/platforms/cpp/odbc/src/config/connection_info.cpp @@ -1103,7 +1103,7 @@ namespace ignite // Bitmask that describes the attributes of a static cursor that are supported by the driver. This // bitmask contains the first subset of attributes; for the second subset, see // SQL_STATIC_CURSOR_ATTRIBUTES2. - intParams[SQL_STATIC_CURSOR_ATTRIBUTES1] = SQL_CA1_NEXT; + intParams[SQL_STATIC_CURSOR_ATTRIBUTES1] = SQL_CA1_NEXT | SQL_CA1_ABSOLUTE; #endif // SQL_STATIC_CURSOR_ATTRIBUTES1 #ifdef SQL_STATIC_CURSOR_ATTRIBUTES2 From 72994434be2cbcf10cb2626f4303395cdac9dcd6 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Thu, 15 Nov 2018 17:08:00 +0300 Subject: [PATCH 099/403] IGNITE-10009: ODBC fix for escaped table names This closes #5080 (cherry picked from commit 665aa9508687e0c37fa81ad0832b60d0e2b7e7b3) --- .../odbc/odbc/OdbcRequestHandler.java | 12 ++---- .../processors/odbc/odbc/OdbcUtils.java | 22 ++++++++++ .../cpp/odbc-test/src/meta_queries_test.cpp | 43 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java index 723ce5c56cc05..10cfd6dbc2f18 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java @@ -868,15 +868,12 @@ private static boolean matchesTableType(String str, String ptrn) { if (str == null) return false; - String pattern = ptrn.toUpperCase().replace("%", ".*").replace("_", "."); + String pattern = OdbcUtils.preprocessPattern(ptrn); String[] types = pattern.split(","); for (String type0 : types) { - String type = type0.trim(); - - if (type.length() >= 2 && type.matches("['\"].*['\"]")) - type = type.substring(1, type.length() - 1); + String type = OdbcUtils.removeQuotationMarksIfNeeded(type0.trim()); if (str.toUpperCase().matches(type)) return true; @@ -899,10 +896,7 @@ private static boolean matches(String str, String ptrn) { if (str == null) return false; - String pattern = ptrn.toUpperCase().replace("%", ".*").replace("_", "."); - - if (pattern.length() >= 2 && pattern.matches("['\"].*['\"]")) - pattern = pattern.substring(1, pattern.length() - 1); + String pattern = OdbcUtils.preprocessPattern(ptrn); return str.toUpperCase().matches(pattern); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java index 966d7bbc4d9ce..d294ac25c583f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.odbc.SqlListenerDataTypes; import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.util.typedef.F; /** * Various ODBC utility methods. @@ -56,6 +57,27 @@ public static String removeQuotationMarksIfNeeded(String str) { return str; } + /** + * Pre-process table or column pattern. + * + * @param ptrn Pattern to pre-process. + * @return Processed pattern. + */ + public static String preprocessPattern(String ptrn) { + if (F.isEmpty(ptrn)) + return ptrn; + + String ptrn0 = ' ' + removeQuotationMarksIfNeeded(ptrn.toUpperCase()); + + ptrn0 = ptrn0.replaceAll("([^\\\\])%", "$1.*"); + + ptrn0 = ptrn0.replaceAll("([^\\\\])_", "$1."); + + ptrn0 = ptrn0.replaceAll("\\\\(.)", "$1"); + + return ptrn0.substring(1); + } + /** * Private constructor. */ diff --git a/modules/platforms/cpp/odbc-test/src/meta_queries_test.cpp b/modules/platforms/cpp/odbc-test/src/meta_queries_test.cpp index d3b4a260625ba..1edda7c124ba0 100644 --- a/modules/platforms/cpp/odbc-test/src/meta_queries_test.cpp +++ b/modules/platforms/cpp/odbc-test/src/meta_queries_test.cpp @@ -418,4 +418,47 @@ BOOST_AUTO_TEST_CASE(TestDdlColumnsMeta) BOOST_REQUIRE_EQUAL(ret, SQL_NO_DATA); } +BOOST_AUTO_TEST_CASE(TestDdlColumnsMetaEscaped) +{ + Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=PUBLIC"); + + SQLCHAR createTable[] = "create table ESG_FOCUS(id int primary key, TEST_COLUMN varchar)"; + SQLRETURN ret = SQLExecDirect(stmt, createTable, SQL_NTS); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + SQLCHAR empty[] = ""; + SQLCHAR table[] = "ESG\\_FOCUS"; + + ret = SQLColumns(stmt, empty, SQL_NTS, empty, SQL_NTS, table, SQL_NTS, empty, SQL_NTS); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + ret = SQLFetch(stmt); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + CheckStringColumn(stmt, 1, ""); + CheckStringColumn(stmt, 2, "\"PUBLIC\""); + CheckStringColumn(stmt, 3, "ESG_FOCUS"); + CheckStringColumn(stmt, 4, "ID"); + + ret = SQLFetch(stmt); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + CheckStringColumn(stmt, 1, ""); + CheckStringColumn(stmt, 2, "\"PUBLIC\""); + CheckStringColumn(stmt, 3, "ESG_FOCUS"); + CheckStringColumn(stmt, 4, "TEST_COLUMN"); + + ret = SQLFetch(stmt); + + BOOST_REQUIRE_EQUAL(ret, SQL_NO_DATA); +} + BOOST_AUTO_TEST_SUITE_END() From 7160acc6ac34202f913430c6e4e3fa75761f82ba Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Fri, 16 Nov 2018 15:40:24 +0300 Subject: [PATCH 100/403] IGNITE-10273: Thin clents now getting the last affinity mapping in all cases. This closes #5399 (cherry picked from commit 64b19a99abdc4e253d0aeb53ba860c8a2f30c51d) --- .../ClientCacheNodePartitionsRequest.java | 3 +-- .../src/cache_client_test.cpp | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java index b9bf80e60608f..377d26fae039a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java @@ -50,8 +50,7 @@ public ClientCacheNodePartitionsRequest(BinaryRawReader reader) { IgniteCache cache = cache(ctx); GridDiscoveryManager discovery = ctx.kernalContext().discovery(); - Collection nodes = discovery.cacheNodes(cache.getName(), - new AffinityTopologyVersion(discovery.topologyVersion())); + Collection nodes = discovery.discoCache().cacheNodes(cache.getName()); Affinity aff = ctx.kernalContext().affinity().affinityProxy(cache.getName()); diff --git a/modules/platforms/cpp/thin-client-test/src/cache_client_test.cpp b/modules/platforms/cpp/thin-client-test/src/cache_client_test.cpp index 24ff48f215f7c..4031d4611ef91 100644 --- a/modules/platforms/cpp/thin-client-test/src/cache_client_test.cpp +++ b/modules/platforms/cpp/thin-client-test/src/cache_client_test.cpp @@ -814,6 +814,33 @@ BOOST_AUTO_TEST_CASE(CacheClientDefaultDynamicCache) } } +BOOST_AUTO_TEST_CASE(CacheClientDefaultDynamicCacheThreeNodes) +{ + StartNode("node1"); + StartNode("node2"); + + IgniteClientConfiguration cfg; + cfg.SetEndPoints("127.0.0.1:11110..11120"); + + IgniteClient client = IgniteClient::Start(cfg); + + cache::CacheClient cache = + client.CreateCache("defaultdynamic3"); + + cache.RefreshAffinityMapping(); + + for (int64_t i = 1; i < 1000; ++i) + cache.Put(ignite::common::LexicalCast(i * 39916801), i * 5039); + + for (int64_t i = 1; i < 1000; ++i) + { + int64_t val; + LocalPeek(cache, ignite::common::LexicalCast(i * 39916801), val); + + BOOST_CHECK_EQUAL(val, i * 5039); + } +} + BOOST_AUTO_TEST_CASE(CacheClientGetAllContainers) { IgniteClientConfiguration cfg; From 56f26b698b9cba789bfce71f6048e432c8cb8f31 Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Fri, 16 Nov 2018 18:56:28 +0300 Subject: [PATCH 101/403] IGNITE-12364 Reworked Sending Full Message logging - Fixes #5413. Signed-off-by: Alexey Goncharuk (cherry picked from commit 34c059a15b3505c8937045b2b973874286bc148a) --- .../GridDhtPartitionsExchangeFuture.java | 10 ---------- .../communication/tcp/TcpCommunicationSpi.java | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 122d835dc1977..4c080682aec31 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -1922,8 +1922,6 @@ private void sendAllPartitions( .map(singleMessage -> fullMsg.copy().joinedNodeAffinity(affinityForJoinedNodes)) .orElse(null); - long time = System.currentTimeMillis(); - // Prepare and send full messages for given nodes. nodes.stream() .map(node -> { @@ -1976,9 +1974,6 @@ private void sendAllPartitions( U.error(log, "Failed to send partitions [node=" + node + ']', e); } }); - - if (log.isInfoEnabled()) - log.info("Sending Full Message performed in " + (System.currentTimeMillis() - time) + " ms."); } /** @@ -3389,16 +3384,11 @@ else if (forceAffReassignment) nodes.addAll(sndResNodes); } - time = System.currentTimeMillis(); - if (!nodes.isEmpty()) sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff); partitionsSent = true; - if (log.isInfoEnabled()) - log.info("Sending Full Message to all nodes performed in " + (System.currentTimeMillis() - time) + " ms."); - if (!stateChangeExchange()) onDone(exchCtx.events().topologyVersion(), null); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java index 877d2be0f2fc8..4bd872ab39e68 100755 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java @@ -269,6 +269,9 @@ @IgniteSpiMultipleInstancesSupport(true) @IgniteSpiConsistencyChecked(optional = false) public class TcpCommunicationSpi extends IgniteSpiAdapter implements CommunicationSpi { + /** Time threshold to log too long connection establish. */ + private static final int CONNECTION_ESTABLISH_THRESHOLD_MS = 100; + /** IPC error message. */ public static final String OUT_OF_RESOURCES_TCP_MSG = "Failed to allocate shared memory segment " + "(switching to TCP, may be slower)."; @@ -2984,10 +2987,18 @@ else if (log.isDebugEnabled()) connectGate.enter(); try { + final long start = System.currentTimeMillis(); + GridCommunicationClient client = createTcpClient(node, connIdx); - if (log.isDebugEnabled()) - log.debug("TCP client created: " + client); + final long time = System.currentTimeMillis() - start; + + if (time > CONNECTION_ESTABLISH_THRESHOLD_MS) { + if (log.isInfoEnabled()) + log.info("TCP client created [client=" + client + ", duration=" + time + "ms]"); + } + else if (log.isDebugEnabled()) + log.debug("TCP client created [client=" + client + ", duration=" + time + "ms]"); return client; } From da9d543a4f98b3ebec5c66801f72db5822522f23 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Mon, 19 Nov 2018 16:47:12 +0700 Subject: [PATCH 102/403] IGNITE-10320 Web Console: Workaround for memory leak in chart component. (cherry picked from commit 166e87ed19929b9e064732752d5a5ce1069b5076) --- .../app/components/ignite-chart/controller.js | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/modules/web-console/frontend/app/components/ignite-chart/controller.js b/modules/web-console/frontend/app/components/ignite-chart/controller.js index 79156fc8110c1..81ed273f6882a 100644 --- a/modules/web-console/frontend/app/components/ignite-chart/controller.js +++ b/modules/web-console/frontend/app/components/ignite-chart/controller.js @@ -21,22 +21,13 @@ import _ from 'lodash'; * @typedef {{x: number, y: {[key: string]: number}}} IgniteChartDataPoint */ -const RANGE_RATE_PRESET = [{ - label: '1 min', - value: 1 -}, { - label: '5 min', - value: 5 -}, { - label: '10 min', - value: 10 -}, { - label: '15 min', - value: 15 -}, { - label: '30 min', - value: 30 -}]; +const RANGE_RATE_PRESET = [ + {label: '1 min', value: 1}, + {label: '5 min', value: 5}, + {label: '10 min', value: 10}, + {label: '15 min', value: 15}, + {label: '30 min', value: 30} +]; export class IgniteChartController { /** @type {import('chart.js').ChartConfiguration} */ @@ -227,7 +218,8 @@ export class IgniteChartController { duration: this.currentRange.value * 1000 * 60, frameRate: 1000 / this.refreshRate || 1 / 3, refresh: this.refreshRate || 3000, - ttl: this.maxRangeInMilliseconds, + // Temporary workaround before https://github.com/nagix/chartjs-plugin-streaming/issues/53 resolved. + // ttl: this.maxRangeInMilliseconds, onRefresh: () => { this.onRefresh(); } @@ -308,6 +300,24 @@ export class IgniteChartController { this.config.data.datasets[datasetIndex].fill = false; } }); + + // Temporary workaround before https://github.com/nagix/chartjs-plugin-streaming/issues/53 resolved. + this.pruneHistory(); + } + + // Temporary workaround before https://github.com/nagix/chartjs-plugin-streaming/issues/53 resolved. + pruneHistory() { + if (!this.xRangeUpdateInProgress) { + const currenTime = Date.now(); + + while (currenTime - this.localHistory[0].x > this.maxRangeInMilliseconds) + this.localHistory.shift(); + + this.config.data.datasets.forEach((dataset) => { + while (currenTime - dataset.data[0].x > this.maxRangeInMilliseconds) + dataset.data.shift(); + }); + } } /** @@ -346,6 +356,8 @@ export class IgniteChartController { changeXRange(range) { if (this.chart) { + this.xRangeUpdateInProgress = true; + this.chart.config.options.plugins.streaming.duration = range.value * 60 * 1000; this.clearDatasets(); @@ -353,6 +365,8 @@ export class IgniteChartController { this.onRefresh(); this.rerenderChart(); + + this.xRangeUpdateInProgress = false; } } From 46288ace9aa193b5a328fbaec8b7248256c72e8c Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Mon, 19 Nov 2018 16:41:28 +0300 Subject: [PATCH 103/403] IGNITE-10285 Fixed U.doInParallel may lead to deadlock - Fixes #5404. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit acfdcdaa0f667b1ebebc26cea90df030804056a7) Signed-off-by: Dmitriy Govorukhin --- .../ignite/internal/util/IgniteUtils.java | 130 ++++++++++++++-- .../internal/util/IgniteUtilsSelfTest.java | 147 +++++++++++++++++- 2 files changed, 257 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index a06a35d2e1f0e..4080924883715 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -10796,40 +10796,92 @@ public static Collection doInParallel( ) throws IgniteCheckedException, IgniteInterruptedCheckedException { if(srcDatas.isEmpty()) return Collections.emptyList(); + int[] batchSizes = calculateOptimalBatchSizes(parallelismLvl, srcDatas.size()); - List> batches = new ArrayList<>(batchSizes.length); + List> batches = new ArrayList<>(batchSizes.length); + + // Set for sharing batches between executor and current thread. + // If executor cannot perform immediately, we will execute task in the current thread. + Set> sharedBatchesSet = new GridConcurrentHashSet<>(batchSizes.length); Iterator iterator = srcDatas.iterator(); - for (int batchSize : batchSizes) { - List batch = new ArrayList<>(batchSize); + for (int idx = 0; idx < batchSizes.length; idx++) { + int batchSize = batchSizes[idx]; + + Batch batch = new Batch<>(batchSize); for (int i = 0; i < batchSize; i++) - batch.add(iterator.next()); + batch.addTask(iterator.next()); batches.add(batch); } - List>> consumerFutures = batches.stream() - .filter(batch -> !batch.isEmpty()) - .map(batch -> executorSvc.submit(() -> { - Collection results = new ArrayList<>(batch.size()); + batches = batches.stream() + .filter(batch -> !batch.tasks.isEmpty()) + // Add to set only after check that batch is not empty. + .peek(sharedBatchesSet::add) + // Setup future in batch for waiting result. + .peek(batch -> batch.future = executorSvc.submit(() -> { + // Batch was stolen by the main stream. + if (!sharedBatchesSet.remove(batch)) { + return null; + } + + Collection results = new ArrayList<>(batch.tasks.size()); - for (T item : batch) + for (T item : batch.tasks) results.add(operation.accept(item)); return results; })) .collect(Collectors.toList()); - Throwable error =null; + Throwable error = null; + + // Stealing jobs if executor is busy and cannot process task immediately. + // Perform batches in a current thread. + for (Batch batch : sharedBatchesSet) { + // Executor steal task. + if (!sharedBatchesSet.remove(batch)) + continue; + + Collection res = new ArrayList<>(batch.tasks.size()); + + try { + for (T item : batch.tasks) + res.add(operation.accept(item)); + + batch.result(res); + } + catch (IgniteCheckedException e) { + batch.result(e); + } + } + // Final result collection. Collection results = new ArrayList<>(srcDatas.size()); - for (Future> future : consumerFutures) { + for (Batch batch: batches) { try { - results.addAll(future.get()); + Throwable err = batch.error; + + if (err != null) { + if (error == null) + error = err; + else + error.addSuppressed(err); + + continue; + } + + Collection res = batch.result(); + + if (res != null) + results.addAll(res); + else + assert error != null; } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -10866,6 +10918,60 @@ public static Collection doInParallel( return results; } + /** + * The batch of tasks with a batch index in global array. + */ + private static class Batch { + /** List tasks. */ + private final List tasks; + + /** */ + private Collection result; + + /** */ + private Throwable error; + + /** */ + private Future> future; + + /** + * @param batchSize Batch size. + */ + private Batch(int batchSize) { + this.tasks = new ArrayList<>(batchSize); + } + + /** + * @param task Add task. + */ + public void addTask(T task){ + tasks.add(task); + } + + /** + * @param res Setup results for tasks. + */ + public void result(Collection res) { + this.result = res; + } + + /** + * @param e Throwable if task was completed with error. + */ + public void result(Throwable e) { + this.error = e; + } + + /** + * Get tasks results. + */ + public Collection result() throws ExecutionException, InterruptedException { + assert future != null; + + return result != null ? result : future.get(); + } + } + /** * Split number of tasks into optimized batches. * @param parallelismLvl Level of parallelism. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java index eea11b6a8be43..73fb9cc9839e7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java @@ -44,20 +44,27 @@ import java.util.List; import java.util.Random; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.cluster.ClusterGroup; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobAdapter; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.util.lang.GridPeerDeployAware; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; @@ -70,7 +77,9 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.testframework.junits.common.GridCommonTest; import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import static java.util.Arrays.asList; import static org.junit.Assert.assertArrayEquals; /** @@ -335,7 +344,7 @@ private SelfReferencedJob(Ignite ignite) throws IgniteCheckedException { arr = new SelfReferencedJob[] {this, this}; - col = Arrays.asList(this, this, this); + col = asList(this, this, this); newContext(); @@ -876,10 +885,10 @@ public void testIsOldestNodeVersionAtLeast() { TcpDiscoveryNode node250ts = new TcpDiscoveryNode(); node250ts.version(v250ts); - assertTrue(U.isOldestNodeVersionAtLeast(v240, Arrays.asList(node240, node241, node250, node250ts))); - assertFalse(U.isOldestNodeVersionAtLeast(v241, Arrays.asList(node240, node241, node250, node250ts))); - assertTrue(U.isOldestNodeVersionAtLeast(v250, Arrays.asList(node250, node250ts))); - assertTrue(U.isOldestNodeVersionAtLeast(v250ts, Arrays.asList(node250, node250ts))); + assertTrue(U.isOldestNodeVersionAtLeast(v240, asList(node240, node241, node250, node250ts))); + assertFalse(U.isOldestNodeVersionAtLeast(v241, asList(node240, node241, node250, node250ts))); + assertTrue(U.isOldestNodeVersionAtLeast(v250, asList(node250, node250ts))); + assertTrue(U.isOldestNodeVersionAtLeast(v250ts, asList(node250, node250ts))); } /** @@ -893,7 +902,7 @@ public void testDoInParallel() throws Throwable { try { IgniteUtils.doInParallel(3, executorService, - Arrays.asList(1, 2, 3), + asList(1, 2, 3), i -> { try { barrier.await(1, TimeUnit.SECONDS); @@ -921,7 +930,7 @@ public void testDoInParallelBatch() { try { IgniteUtils.doInParallel(2, executorService, - Arrays.asList(1, 2, 3), + asList(1, 2, 3), i -> { try { barrier.await(400, TimeUnit.MILLISECONDS); @@ -989,6 +998,128 @@ public void testDoInParallelResultsOrder() throws IgniteCheckedException { } } + /** + * Test parallel execution steal job. + */ + public void testDoInParallelWithStealingJob() throws IgniteCheckedException { + // Pool size should be less that input data collection. + ExecutorService executorService = Executors.newFixedThreadPool(1); + + CountDownLatch mainThreadLatch = new CountDownLatch(1); + CountDownLatch poolThreadLatch = new CountDownLatch(1); + + // Busy one thread from the pool. + executorService.submit(new Runnable() { + @Override + public void run() { + try { + poolThreadLatch.await(); + } + catch (InterruptedException e) { + throw new IgniteInterruptedException(e); + } + } + }); + + List data = asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + AtomicInteger taskProcessed = new AtomicInteger(); + + long threadId = Thread.currentThread().getId(); + + AtomicInteger curThreadCnt = new AtomicInteger(); + AtomicInteger poolThreadCnt = new AtomicInteger(); + + Collection res = U.doInParallel(10, + executorService, + data, + new IgniteThrowableConsumer() { + @Override public Integer accept(Integer cnt) throws IgniteInterruptedCheckedException { + // Release thread in pool in the middle of range. + if (taskProcessed.getAndIncrement() == (data.size() / 2) - 1) { + poolThreadLatch.countDown(); + + try { + // Await thread in thread pool complete task. + mainThreadLatch.await(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + throw new IgniteInterruptedCheckedException(e); + } + } + + // Increment if executed in current thread. + if (Thread.currentThread().getId() == threadId) + curThreadCnt.incrementAndGet(); + else { + poolThreadCnt.incrementAndGet(); + + if (taskProcessed.get() == data.size()) + mainThreadLatch.countDown(); + } + + return -cnt; + } + }); + + Assert.assertEquals(curThreadCnt.get() + poolThreadCnt.get(), data.size()); + Assert.assertEquals(5, curThreadCnt.get()); + Assert.assertEquals(5, poolThreadCnt.get()); + Assert.assertEquals(asList(0, -1, -2, -3, -4, -5, -6, -7, -8, -9), res); + } + + /** + * Test parallel execution steal job. + */ + public void testDoInParallelWithStealingJobRunTaskInExecutor() throws Exception { + // Pool size should be less that input data collection. + ExecutorService executorService = Executors.newFixedThreadPool(2); + + Future f1 = executorService.submit(()-> runTask(executorService)); + Future f2 = executorService.submit(()-> runTask(executorService)); + Future f3 = executorService.submit(()-> runTask(executorService)); + + f1.get(); + f2.get(); + f3.get(); + } + + /** + * + * @param executorService Executor service. + */ + private void runTask(ExecutorService executorService) { + List data = asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + long threadId = Thread.currentThread().getId(); + + AtomicInteger curThreadCnt = new AtomicInteger(); + + Collection res; + + try { + res = U.doInParallel(10, + executorService, + data, + new IgniteThrowableConsumer() { + @Override public Integer accept(Integer cnt) { + if (Thread.currentThread().getId() == threadId) + curThreadCnt.incrementAndGet(); + + return -cnt; + } + }); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + + Assert.assertTrue(curThreadCnt.get() > 0); + Assert.assertEquals(asList(0, -1, -2, -3, -4, -5, -6, -7, -8, -9), res); + } + /** * Template method to test parallel execution * @param executorService ExecutorService. @@ -1031,7 +1162,7 @@ public void testDoInParallelException() { IgniteUtils.doInParallel( 1, executorService, - Arrays.asList(1, 2, 3), + asList(1, 2, 3), i -> { if (Integer.valueOf(1).equals(i)) throw new IgniteCheckedException(expectedException); From 40df3f3b5481f5af5a10bdcec5cc572c710ea1a7 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Mon, 19 Nov 2018 18:48:40 +0300 Subject: [PATCH 104/403] IGNITE-9558 Avoid blocking transactions on client connect when possible - Fixes #4933. (cherry picked from commit a63a81a51418a8ae1dadff019d9e5eace38e1631) --- .../ignite/codegen/MessageCodeGenerator.java | 11 +- .../ignite/internal/GridJobCancelRequest.java | 2 +- .../internal/GridJobExecuteRequest.java | 4 +- .../internal/GridJobExecuteResponse.java | 4 +- .../internal/GridJobSiblingsRequest.java | 2 +- .../internal/GridJobSiblingsResponse.java | 2 +- .../internal/GridTaskCancelRequest.java | 2 +- .../internal/GridTaskSessionRequest.java | 2 +- .../internal/direct/DirectMessageReader.java | 30 ++ .../internal/direct/DirectMessageWriter.java | 26 ++ .../direct/stream/DirectByteBufferStream.java | 11 + .../v1/DirectByteBufferStreamImplV1.java | 13 + .../v2/DirectByteBufferStreamImplV2.java | 19 +- .../v3/DirectByteBufferStreamImplV3.java | 298 ++++++++++++++++++ .../checkpoint/GridCheckpointRequest.java | 2 +- .../managers/communication/GridIoManager.java | 2 +- .../communication/GridIoUserMessage.java | 2 +- .../communication/IgniteIoTestMessage.java | 2 + .../deployment/GridDeploymentInfoBean.java | 2 +- .../deployment/GridDeploymentRequest.java | 2 +- .../deployment/GridDeploymentResponse.java | 2 +- .../eventstorage/GridEventStorageMessage.java | 2 +- .../affinity/AffinityTopologyVersion.java | 2 +- .../affinity/GridAffinityAssignmentCache.java | 37 ++- .../UserAuthenticateResponseMessage.java | 2 +- .../cache/CacheEntryInfoCollection.java | 2 +- .../processors/cache/CacheEntryPredicate.java | 2 +- .../cache/CacheEntryPredicateAdapter.java | 2 +- .../CacheEntryPredicateContainsValue.java | 2 +- .../cache/CacheEntryPredicateHasValue.java | 2 +- .../cache/CacheEntryPredicateNoValue.java | 2 +- .../CacheEntrySerializablePredicate.java | 2 +- .../processors/cache/CacheEvictionEntry.java | 2 +- .../cache/CacheInvokeDirectResult.java | 2 +- .../processors/cache/CacheObject.java | 2 +- .../processors/cache/CacheObjectAdapter.java | 2 +- .../cache/CacheObjectByteArrayImpl.java | 2 +- .../processors/cache/CacheObjectImpl.java | 2 +- .../cache/GridCacheAffinityManager.java | 14 +- .../cache/GridCacheGroupIdMessage.java | 6 +- .../processors/cache/GridCacheIdMessage.java | 6 +- .../processors/cache/GridCacheIoManager.java | 17 +- .../processors/cache/GridCacheMessage.java | 41 ++- .../GridCachePartitionExchangeManager.java | 202 +++++++++--- .../GridChangeGlobalStateMessageResponse.java | 10 +- .../processors/cache/KeyCacheObject.java | 2 +- .../processors/cache/WalStateAckMessage.java | 2 + .../GridCacheTtlUpdateRequest.java | 30 +- .../GridCacheTxRecoveryRequest.java | 26 +- .../GridCacheTxRecoveryResponse.java | 14 +- .../GridDistributedBaseMessage.java | 16 +- .../GridDistributedLockRequest.java | 54 ++-- .../GridDistributedLockResponse.java | 14 +- .../GridDistributedTxFinishRequest.java | 62 ++-- .../GridDistributedTxFinishResponse.java | 18 +- .../GridDistributedTxPrepareRequest.java | 54 ++-- .../GridDistributedTxPrepareResponse.java | 14 +- .../GridDistributedUnlockRequest.java | 6 +- .../dht/ClientCacheDhtTopologyFuture.java | 5 + .../dht/GridDhtAffinityAssignmentRequest.java | 18 +- .../GridDhtAffinityAssignmentResponse.java | 26 +- .../distributed/dht/GridDhtCacheAdapter.java | 5 + .../distributed/dht/GridDhtLockRequest.java | 46 +-- .../distributed/dht/GridDhtLockResponse.java | 18 +- .../dht/GridDhtTopologyFuture.java | 6 + .../dht/GridDhtTransactionalCacheAdapter.java | 3 +- .../dht/GridDhtTxFinishRequest.java | 54 ++-- .../dht/GridDhtTxFinishResponse.java | 14 +- .../GridDhtTxOnePhaseCommitAckRequest.java | 6 +- .../dht/GridDhtTxPrepareRequest.java | 96 +++--- .../dht/GridDhtTxPrepareResponse.java | 22 +- .../dht/GridDhtTxQueryEnlistRequest.java | 30 +- .../dht/GridDhtTxQueryEnlistResponse.java | 14 +- .../dht/GridDhtTxQueryFirstEnlistRequest.java | 42 +-- .../distributed/dht/GridDhtUnlockRequest.java | 6 +- .../dht/PartitionUpdateCountersMessage.java | 2 + .../GridDhtAtomicAbstractUpdateRequest.java | 42 +-- .../GridDhtAtomicDeferredUpdateResponse.java | 6 +- .../dht/atomic/GridDhtAtomicNearResponse.java | 22 +- .../GridDhtAtomicSingleUpdateRequest.java | 18 +- .../atomic/GridDhtAtomicUpdateRequest.java | 66 ++-- .../atomic/GridDhtAtomicUpdateResponse.java | 18 +- .../GridNearAtomicAbstractUpdateRequest.java | 34 +- .../GridNearAtomicCheckUpdateRequest.java | 10 +- .../GridNearAtomicFullUpdateRequest.java | 38 +-- ...idNearAtomicSingleUpdateFilterRequest.java | 6 +- ...idNearAtomicSingleUpdateInvokeRequest.java | 10 +- .../GridNearAtomicSingleUpdateRequest.java | 10 +- .../atomic/GridNearAtomicUpdateResponse.java | 34 +- .../preloader/GridDhtForceKeysRequest.java | 22 +- .../preloader/GridDhtForceKeysResponse.java | 22 +- .../GridDhtPartitionDemandLegacyMessage.java | 40 +-- .../GridDhtPartitionDemandMessage.java | 32 +- .../preloader/GridDhtPartitionExchangeId.java | 6 +- .../GridDhtPartitionSupplyMessage.java | 44 +-- .../GridDhtPartitionSupplyMessageV2.java | 6 +- .../GridDhtPartitionsAbstractMessage.java | 14 +- .../GridDhtPartitionsExchangeFuture.java | 10 + .../GridDhtPartitionsFullMessage.java | 58 ++-- .../GridDhtPartitionsSingleMessage.java | 64 ++-- .../GridDhtPartitionsSingleRequest.java | 6 +- .../dht/preloader/latch/LatchAckMessage.java | 6 +- .../GridDhtPartitionTopologyImpl.java | 112 ++++--- .../distributed/near/CacheVersionedValue.java | 2 +- .../distributed/near/GridNearGetRequest.java | 72 ++--- .../distributed/near/GridNearGetResponse.java | 34 +- .../distributed/near/GridNearLockRequest.java | 42 +-- .../near/GridNearLockResponse.java | 30 +- .../near/GridNearSingleGetRequest.java | 54 ++-- .../near/GridNearSingleGetResponse.java | 26 +- .../near/GridNearTxEnlistRequest.java | 70 ++-- .../near/GridNearTxEnlistResponse.java | 34 +- .../near/GridNearTxFinishRequest.java | 10 +- .../near/GridNearTxFinishResponse.java | 14 +- .../near/GridNearTxPrepareRequest.java | 30 +- .../near/GridNearTxPrepareResponse.java | 70 ++-- .../near/GridNearTxQueryEnlistRequest.java | 78 ++--- .../near/GridNearTxQueryEnlistResponse.java | 44 +-- .../GridNearTxQueryResultsEnlistRequest.java | 62 ++-- .../GridNearTxQueryResultsEnlistResponse.java | 12 +- .../near/GridNearUnlockRequest.java | 2 +- .../mvcc/msg/MvccAckRequestTxAndQueryId.java | 1 + .../cache/mvcc/msg/MvccSnapshotResponse.java | 2 + .../cache/mvcc/msg/MvccTxSnapshotRequest.java | 2 + .../PartitionCountersNeighborcastRequest.java | 10 +- ...PartitionCountersNeighborcastResponse.java | 6 +- .../cache/query/GridCacheQueryRequest.java | 112 +++---- .../cache/query/GridCacheQueryResponse.java | 26 +- .../CacheContinuousQueryBatchAck.java | 16 +- .../continuous/CacheContinuousQueryEntry.java | 4 +- .../cache/transactions/IgniteTxHandler.java | 5 + .../cache/transactions/IgniteTxKey.java | 2 +- .../cache/transactions/TxLocksRequest.java | 10 +- .../cache/transactions/TxLocksResponse.java | 18 +- .../version/GridCacheRawVersionedEntry.java | 4 +- .../continuous/GridContinuousMessage.java | 2 +- .../datastreamer/DataStreamerRequest.java | 4 +- .../datastreamer/DataStreamerResponse.java | 2 +- .../shuffle/HadoopDirectShuffleMessage.java | 4 +- .../hadoop/shuffle/HadoopShuffleMessage.java | 2 + .../processors/igfs/IgfsAckMessage.java | 2 +- .../processors/igfs/IgfsBlockKey.java | 2 +- .../processors/igfs/IgfsBlocksMessage.java | 2 +- .../igfs/IgfsCommunicationMessage.java | 2 +- .../processors/igfs/IgfsDeleteMessage.java | 2 +- .../igfs/IgfsFragmentizerRequest.java | 2 +- .../igfs/IgfsFragmentizerResponse.java | 2 +- .../processors/igfs/IgfsSyncMessage.java | 2 +- .../messages/GridQueryNextPageResponse.java | 4 +- .../message/SchemaOperationStatusMessage.java | 2 + .../handlers/task/GridTaskResultRequest.java | 2 +- .../handlers/task/GridTaskResultResponse.java | 2 +- .../internal/util/GridByteArrayList.java | 2 +- .../ignite/internal/util/GridIntList.java | 2 +- .../ignite/internal/util/GridLongList.java | 2 + .../MessageCollectionItemType.java | 5 +- .../communication/MessageReader.java | 9 + .../communication/MessageWriter.java | 10 + .../jobstealing/JobStealingRequest.java | 2 +- .../tcp/messages/HandshakeMessage.java | 2 + .../tcp/messages/HandshakeMessage2.java | 2 + .../tcp/messages/NodeIdMessage.java | 2 + .../messages/RecoveryLastReceivedMessage.java | 2 + .../cache/CacheNoAffinityExchangeTest.java | 218 +++++++++++++ .../IgniteClusterActivateDeactivateTest.java | 3 +- .../distributed/CacheExchangeMergeTest.java | 2 + .../testsuites/IgniteCacheTestSuite6.java | 3 + .../processors/cache/query/QueryTable.java | 2 + .../twostep/msg/GridH2IndexRangeRequest.java | 2 + .../twostep/msg/GridH2IndexRangeResponse.java | 2 + .../h2/twostep/msg/GridH2QueryRequest.java | 6 +- .../h2/twostep/msg/GridH2ValueMessage.java | 2 + 172 files changed, 2222 insertions(+), 1288 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java index 7492e51977b11..ca2dcdc2b98f7 100644 --- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java +++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java @@ -43,7 +43,7 @@ import org.apache.ignite.internal.GridDirectMap; import org.apache.ignite.internal.GridDirectTransient; import org.apache.ignite.internal.IgniteCodeGeneratingFail; -import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccRecoveryFinishedMessage; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; @@ -116,6 +116,7 @@ public class MessageCodeGenerator { TYPES.put(BitSet.class, MessageCollectionItemType.BIT_SET); TYPES.put(UUID.class, MessageCollectionItemType.UUID); TYPES.put(IgniteUuid.class, MessageCollectionItemType.IGNITE_UUID); + TYPES.put(AffinityTopologyVersion.class, MessageCollectionItemType.AFFINITY_TOPOLOGY_VERSION); } /** @@ -169,9 +170,7 @@ public static void main(String[] args) throws Exception { // gen.generateAll(true); - gen.generateAndWrite(MvccRecoveryFinishedMessage.class); - -// gen.generateAndWrite(GridNearAtomicUpdateRequest.class); +// gen.generateAndWrite(GridCacheMessage.class); // gen.generateAndWrite(GridMessageCollection.class); // gen.generateAndWrite(DataStreamerEntry.class); @@ -663,6 +662,8 @@ else if (type == UUID.class) returnFalseIfFailed(write, "writer.writeUuid", field, getExpr); else if (type == IgniteUuid.class) returnFalseIfFailed(write, "writer.writeIgniteUuid", field, getExpr); + else if (type == AffinityTopologyVersion.class) + returnFalseIfFailed(write, "writer.writeAffinityTopologyVersion", field, getExpr); else if (type.isEnum()) { String arg = getExpr + " != null ? (byte)" + getExpr + ".ordinal() : -1"; @@ -745,6 +746,8 @@ else if (type == UUID.class) returnFalseIfReadFailed(name, "reader.readUuid", setExpr, field); else if (type == IgniteUuid.class) returnFalseIfReadFailed(name, "reader.readIgniteUuid", setExpr, field); + else if (type == AffinityTopologyVersion.class) + returnFalseIfReadFailed(name, "reader.readAffinityTopologyVersion", setExpr, field); else if (type.isEnum()) { String loc = name + "Ord"; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java index aaa69eaff7190..ac3a87336fc9f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java @@ -201,4 +201,4 @@ public boolean system() { @Override public String toString() { return S.toString(GridJobCancelRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java index 4357d1da07dce..ebfeb0153ffdc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java @@ -664,7 +664,7 @@ public AffinityTopologyVersion getTopVer() { writer.incrementState(); case 24: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -885,7 +885,7 @@ public AffinityTopologyVersion getTopVer() { reader.incrementState(); case 24: - topVer = reader.readMessage("topVer"); + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java index 312435e922750..f052edf07d446 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java @@ -282,7 +282,7 @@ public AffinityTopologyVersion getRetryTopologyVersion() { writer.incrementState(); case 6: - if (!writer.writeMessage("retry", retry)) + if (!writer.writeAffinityTopologyVersion("retry", retry)) return false; writer.incrementState(); @@ -355,7 +355,7 @@ public AffinityTopologyVersion getRetryTopologyVersion() { reader.incrementState(); case 6: - retry = reader.readMessage("retry"); + retry = reader.readAffinityTopologyVersion("retry"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java index 8a11cef33aa21..d743a355f8d8e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java @@ -161,4 +161,4 @@ public byte[] topicBytes() { @Override public String toString() { return S.toString(GridJobSiblingsRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java index 3911446d2b86b..dc59ab5f3057c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java @@ -141,4 +141,4 @@ public void unmarshalSiblings(Marshaller marsh) throws IgniteCheckedException { @Override public String toString() { return S.toString(GridJobSiblingsResponse.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java index 273d0a777a468..71c318b537dd5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java @@ -124,4 +124,4 @@ public IgniteUuid sessionId() { @Override public String toString() { return S.toString(GridTaskCancelRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java index dbac893189e38..576392e097fc3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java @@ -189,4 +189,4 @@ public IgniteUuid getJobId() { @Override public String toString() { return S.toString(GridTaskSessionRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java index 47d7877816824..b8208016aba53 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java @@ -27,6 +27,9 @@ import org.apache.ignite.internal.direct.stream.DirectByteBufferStream; import org.apache.ignite.internal.direct.stream.v1.DirectByteBufferStreamImplV1; import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2; +import org.apache.ignite.internal.direct.stream.v3.DirectByteBufferStreamImplV3; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteOutClosure; import org.apache.ignite.lang.IgniteUuid; @@ -41,8 +44,13 @@ */ public class DirectMessageReader implements MessageReader { /** State. */ + @GridToStringInclude private final DirectMessageState state; + /** Protocol version. */ + @GridToStringInclude + private final byte protoVer; + /** Whether last field was fully read. */ private boolean lastRead; @@ -56,6 +64,8 @@ public DirectMessageReader(final MessageFactory msgFactory, final byte protoVer) return new StateItem(msgFactory, protoVer); } }); + + this.protoVer = protoVer; } /** {@inheritDoc} */ @@ -304,6 +314,21 @@ public DirectMessageReader(final MessageFactory msgFactory, final byte protoVer) return val; } + /** {@inheritDoc} */ + @Override public AffinityTopologyVersion readAffinityTopologyVersion(String name) { + if (protoVer >= 3) { + DirectByteBufferStream stream = state.item().stream; + + AffinityTopologyVersion val = stream.readAffinityTopologyVersion(); + + lastRead = stream.lastFinished(); + + return val; + } + + return readMessage(name); + } + /** {@inheritDoc} */ @Nullable @Override public T readMessage(String name) { DirectByteBufferStream stream = state.item().stream; @@ -409,6 +434,11 @@ public StateItem(MessageFactory msgFactory, byte protoVer) { break; + case 3: + stream = new DirectByteBufferStreamImplV3(msgFactory); + + break; + default: throw new IllegalStateException("Invalid protocol version: " + protoVer); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java index 51cea174e9a1d..bb88ffc851669 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java @@ -27,6 +27,8 @@ import org.apache.ignite.internal.direct.stream.DirectByteBufferStream; import org.apache.ignite.internal.direct.stream.v1.DirectByteBufferStreamImplV1; import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2; +import org.apache.ignite.internal.direct.stream.v3.DirectByteBufferStreamImplV3; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteOutClosure; @@ -44,6 +46,10 @@ public class DirectMessageWriter implements MessageWriter { @GridToStringInclude private final DirectMessageState state; + /** Protocol version. */ + @GridToStringInclude + private final byte protoVer; + /** * @param protoVer Protocol version. */ @@ -53,6 +59,8 @@ public DirectMessageWriter(final byte protoVer) { return new StateItem(protoVer); } }); + + this.protoVer = protoVer; } /** {@inheritDoc} */ @@ -272,6 +280,19 @@ public DirectMessageWriter(final byte protoVer) { return stream.lastFinished(); } + /** {@inheritDoc} */ + @Override public boolean writeAffinityTopologyVersion(String name, AffinityTopologyVersion val) { + if (protoVer >= 3) { + DirectByteBufferStream stream = state.item().stream; + + stream.writeAffinityTopologyVersion(val); + + return stream.lastFinished(); + } + + return writeMessage(name, val); + } + /** {@inheritDoc} */ @Override public boolean writeMessage(String name, @Nullable Message msg) { DirectByteBufferStream stream = state.item().stream; @@ -376,6 +397,11 @@ public StateItem(byte protoVer) { break; + case 3: + stream = new DirectByteBufferStreamImplV3(null); + + break; + default: throw new IllegalStateException("Invalid protocol version: " + protoVer); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java index 204e6b034530b..ae5502eb75f8e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Map; import java.util.UUID; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; @@ -160,6 +161,11 @@ public interface DirectByteBufferStream { */ public void writeIgniteUuid(IgniteUuid val); + /** + * @param val Value. + */ + public void writeAffinityTopologyVersion(AffinityTopologyVersion val); + /** * @param msg Message. * @param writer Writer. @@ -289,6 +295,11 @@ public void writeMap(Map map, MessageCollectionItemType keyType, Me */ public IgniteUuid readIgniteUuid(); + /** + * @return Value. + */ + public AffinityTopologyVersion readAffinityTopologyVersion(); + /** * @param reader Reader. * @return Message. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java index c78c47914fdfe..118e1f1d0cbe7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java @@ -27,6 +27,7 @@ import java.util.NoSuchElementException; import java.util.UUID; import org.apache.ignite.internal.direct.stream.DirectByteBufferStream; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -494,6 +495,11 @@ public DirectByteBufferStreamImplV1(MessageFactory msgFactory) { writeByteArray(val != null ? U.igniteUuidToBytes(val) : null); } + /** {@inheritDoc} */ + @Override public void writeAffinityTopologyVersion(AffinityTopologyVersion val) { + throw new UnsupportedOperationException("Not implemented"); + } + /** {@inheritDoc} */ @Override public void writeMessage(Message msg, MessageWriter writer) { if (msg != null) { @@ -811,6 +817,11 @@ public DirectByteBufferStreamImplV1(MessageFactory msgFactory) { return arr != null ? U.bytesToIgniteUuid(arr, 0) : null; } + /** {@inheritDoc} */ + @Override public AffinityTopologyVersion readAffinityTopologyVersion() { + throw new UnsupportedOperationException("Not implemented"); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public T readMessage(MessageReader reader) { @@ -1212,6 +1223,7 @@ private void write(MessageCollectionItemType type, Object val, MessageWriter wri break; + case AFFINITY_TOPOLOGY_VERSION: case MSG: try { if (val != null) @@ -1298,6 +1310,7 @@ private Object read(MessageCollectionItemType type, MessageReader reader) { case IGNITE_UUID: return readIgniteUuid(); + case AFFINITY_TOPOLOGY_VERSION: case MSG: return readMessage(reader); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java index e338bc0187d46..fd93cfb81f832 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java @@ -29,6 +29,7 @@ import java.util.UUID; import org.apache.ignite.IgniteException; import org.apache.ignite.internal.direct.stream.DirectByteBufferStream; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -298,7 +299,7 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream { private long uuidLocId; /** */ - private boolean lastFinished; + protected boolean lastFinished; /** * @param msgFactory Message factory. @@ -657,6 +658,11 @@ public DirectByteBufferStreamImplV2(MessageFactory msgFactory) { } } + /** {@inheritDoc} */ + @Override public void writeAffinityTopologyVersion(AffinityTopologyVersion val) { + throw new UnsupportedOperationException("Not implemented"); + } + /** {@inheritDoc} */ @Override public void writeMessage(Message msg, MessageWriter writer) { if (msg != null) { @@ -1152,6 +1158,11 @@ private void writeRandomAccessList(List list, MessageCollectionItemType i return val; } + /** {@inheritDoc} */ + @Override public AffinityTopologyVersion readAffinityTopologyVersion() { + throw new UnsupportedOperationException("Not implemented"); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public T readMessage(MessageReader reader) { @@ -1587,7 +1598,7 @@ T readArrayLE(ArrayCreator creator, int typeSize, int lenShift, long off) * @param val Value. * @param writer Writer. */ - private void write(MessageCollectionItemType type, Object val, MessageWriter writer) { + protected void write(MessageCollectionItemType type, Object val, MessageWriter writer) { switch (type) { case BYTE: writeByte((Byte)val); @@ -1689,6 +1700,7 @@ private void write(MessageCollectionItemType type, Object val, MessageWriter wri break; + case AFFINITY_TOPOLOGY_VERSION: case MSG: try { if (val != null) @@ -1713,7 +1725,7 @@ private void write(MessageCollectionItemType type, Object val, MessageWriter wri * @param reader Reader. * @return Value. */ - private Object read(MessageCollectionItemType type, MessageReader reader) { + protected Object read(MessageCollectionItemType type, MessageReader reader) { switch (type) { case BYTE: return readByte(); @@ -1775,6 +1787,7 @@ private Object read(MessageCollectionItemType type, MessageReader reader) { case IGNITE_UUID: return readIgniteUuid(); + case AFFINITY_TOPOLOGY_VERSION: case MSG: return readMessage(reader); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java new file mode 100644 index 0000000000000..89043ebcbc0c5 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.direct.stream.v3; + +import java.util.BitSet; +import java.util.UUID; +import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.apache.ignite.plugin.extensions.communication.MessageReader; +import org.apache.ignite.plugin.extensions.communication.MessageWriter; + +/** + * + */ +public class DirectByteBufferStreamImplV3 extends DirectByteBufferStreamImplV2 { + /** */ + private byte topVerState; + + /** */ + private long topVerMajor; + + /** */ + private int topVerMinor; + + /** + * @param msgFactory Message factory. + */ + public DirectByteBufferStreamImplV3(MessageFactory msgFactory) { + super(msgFactory); + } + + /** {@inheritDoc} */ + @Override public void writeAffinityTopologyVersion(AffinityTopologyVersion val) { + if (val != null) { + switch (topVerState) { + case 0: + writeInt(val.minorTopologyVersion()); + + if (!lastFinished) + return; + + topVerState++; + + case 1: + writeLong(val.topologyVersion()); + + if (!lastFinished) + return; + + topVerState = 0; + } + } + else + writeInt(-1); + } + + /** {@inheritDoc} */ + @Override public AffinityTopologyVersion readAffinityTopologyVersion() { + switch (topVerState) { + case 0: + topVerMinor = readInt(); + + if (!lastFinished || topVerMinor == -1) + return null; + + topVerState++; + + case 1: + topVerMajor = readLong(); + + if (!lastFinished) + return null; + + topVerState = 0; + } + + return new AffinityTopologyVersion(topVerMajor, topVerMinor); + } + + /** {@inheritDoc} */ + @Override protected void write(MessageCollectionItemType type, Object val, MessageWriter writer) { + switch (type) { + case BYTE: + writeByte((Byte)val); + + break; + + case SHORT: + writeShort((Short)val); + + break; + + case INT: + writeInt((Integer)val); + + break; + + case LONG: + writeLong((Long)val); + + break; + + case FLOAT: + writeFloat((Float)val); + + break; + + case DOUBLE: + writeDouble((Double)val); + + break; + + case CHAR: + writeChar((Character)val); + + break; + + case BOOLEAN: + writeBoolean((Boolean)val); + + break; + + case BYTE_ARR: + writeByteArray((byte[])val); + + break; + + case SHORT_ARR: + writeShortArray((short[])val); + + break; + + case INT_ARR: + writeIntArray((int[])val); + + break; + + case LONG_ARR: + writeLongArray((long[])val); + + break; + + case FLOAT_ARR: + writeFloatArray((float[])val); + + break; + + case DOUBLE_ARR: + writeDoubleArray((double[])val); + + break; + + case CHAR_ARR: + writeCharArray((char[])val); + + break; + + case BOOLEAN_ARR: + writeBooleanArray((boolean[])val); + + break; + + case STRING: + writeString((String)val); + + break; + + case BIT_SET: + writeBitSet((BitSet)val); + + break; + + case UUID: + writeUuid((UUID)val); + + break; + + case IGNITE_UUID: + writeIgniteUuid((IgniteUuid)val); + + break; + + case AFFINITY_TOPOLOGY_VERSION: + writeAffinityTopologyVersion((AffinityTopologyVersion)val); + + break; + case MSG: + try { + if (val != null) + writer.beforeInnerMessageWrite(); + + writeMessage((Message)val, writer); + } + finally { + if (val != null) + writer.afterInnerMessageWrite(lastFinished); + } + + break; + + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } + + /** {@inheritDoc} */ + @Override protected Object read(MessageCollectionItemType type, MessageReader reader) { + switch (type) { + case BYTE: + return readByte(); + + case SHORT: + return readShort(); + + case INT: + return readInt(); + + case LONG: + return readLong(); + + case FLOAT: + return readFloat(); + + case DOUBLE: + return readDouble(); + + case CHAR: + return readChar(); + + case BOOLEAN: + return readBoolean(); + + case BYTE_ARR: + return readByteArray(); + + case SHORT_ARR: + return readShortArray(); + + case INT_ARR: + return readIntArray(); + + case LONG_ARR: + return readLongArray(); + + case FLOAT_ARR: + return readFloatArray(); + + case DOUBLE_ARR: + return readDoubleArray(); + + case CHAR_ARR: + return readCharArray(); + + case BOOLEAN_ARR: + return readBooleanArray(); + + case STRING: + return readString(); + + case BIT_SET: + return readBitSet(); + + case UUID: + return readUuid(); + + case IGNITE_UUID: + return readIgniteUuid(); + + case AFFINITY_TOPOLOGY_VERSION: + return readAffinityTopologyVersion(); + + case MSG: + return readMessage(reader); + + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java index 8b21ff2fe5382..4b25e0b687662 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java @@ -177,4 +177,4 @@ public String getCheckpointSpi() { @Override public String toString() { return S.toString(GridCheckpointRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java index b3c80b05f635b..ae117bb073558 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java @@ -135,7 +135,7 @@ public class GridIoManager extends GridManagerAdapter CUR_PLC = new ThreadLocal<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java index 332a9de511c79..408fad773f93e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java @@ -358,4 +358,4 @@ public void deployment(GridDeployment dep) { @Override public String toString() { return S.toString(GridIoUserMessage.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java index 0a8b2b7a38e60..a6a2469736ef2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.util.UUID; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageReader; @@ -28,6 +29,7 @@ /** * */ +@IgniteCodeGeneratingFail public class IgniteIoTestMessage implements Message { /** */ private static byte FLAG_PROC_FROM_NIO = 1; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java index 7f58ce36001ab..72f5ec6b30579 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java @@ -277,4 +277,4 @@ public GridDeploymentInfoBean(GridDeploymentInfo dep) { @Override public String toString() { return S.toString(GridDeploymentInfoBean.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java index 729cf4c54278e..708c64860579e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java @@ -278,4 +278,4 @@ public void nodeIds(Collection nodeIds) { @Override public String toString() { return S.toString(GridDeploymentRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java index d1b0384f0fb6a..591957d37bd2f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java @@ -197,4 +197,4 @@ void errorMessage(String errMsg) { @Override public String toString() { return S.toString(GridDeploymentResponse.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java index 515500b91d0f1..fd5326cf0577c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java @@ -445,4 +445,4 @@ void exceptionBytes(byte[] exBytes) { @Override public String toString() { return S.toString(GridEventStorageMessage.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java index 44b27534dee62..333841d1b33be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java @@ -219,4 +219,4 @@ public int minorTopologyVersion() { @Override public String toString() { return S.toString(AffinityTopologyVersion.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java index 2290ce6bee725..d98ed5f507091 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java @@ -691,19 +691,41 @@ public AffinityAssignment readyAffinity(AffinityTopologyVersion topVer) { * @return Cached affinity. */ public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer) { + AffinityTopologyVersion lastAffChangeTopVer = + ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer); + + return cachedAffinity(topVer, lastAffChangeTopVer); + } + + /** + * Get cached affinity for specified topology version. + * + * @param topVer Topology version. + * @return Cached affinity. + */ + public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer, AffinityTopologyVersion lastAffChangeTopVer) { if (topVer.equals(AffinityTopologyVersion.NONE)) - topVer = lastVersion(); - else - awaitTopologyVersion(topVer); + topVer = lastAffChangeTopVer = lastVersion(); + else { + if (lastAffChangeTopVer.equals(AffinityTopologyVersion.NONE)) + lastAffChangeTopVer = topVer; + + awaitTopologyVersion(lastAffChangeTopVer); + } assert topVer.topologyVersion() >= 0 : topVer; AffinityAssignment cache = head.get(); - if (!cache.topologyVersion().equals(topVer)) { - cache = affCache.get(topVer); + if (!(cache.topologyVersion().compareTo(lastAffChangeTopVer) >= 0 && + cache.topologyVersion().compareTo(topVer) <= 0)) { - if (cache == null) { + Map.Entry e = affCache.ceilingEntry(lastAffChangeTopVer); + + if (e != null) + cache = e.getValue(); + + if (cache == null || cache.topologyVersion().compareTo(topVer) > 0) { throw new IllegalStateException("Getting affinity for topology version earlier than affinity is " + "calculated [locNode=" + ctx.discovery().localNode() + ", grp=" + cacheOrGrpName + @@ -714,7 +736,8 @@ public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer) { } } - assert cache.topologyVersion().equals(topVer) : "Invalid cached affinity: " + cache; + assert cache.topologyVersion().compareTo(lastAffChangeTopVer) >= 0 && + cache.topologyVersion().compareTo(topVer) <= 0 : "Invalid cached affinity: [cache=" + cache + ", topVer=" + topVer + ", lastAffChangedTopVer=" + lastAffChangeTopVer + "]"; return cache; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java index d86b1ad91a551..e3dee3ce92a4f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java @@ -98,7 +98,6 @@ public IgniteUuid id() { writer.incrementState(); - } return true; @@ -127,6 +126,7 @@ public IgniteUuid id() { return false; reader.incrementState(); + } return reader.afterMessageRead(UserAuthenticateResponseMessage.class); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java index 614d7c06fbf56..968afd5212640 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java @@ -128,4 +128,4 @@ public void add(GridCacheEntryInfo info) { @Override public byte fieldsCount() { return 1; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java index 61cbb9e04fb54..36312a1591135 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java @@ -42,4 +42,4 @@ public interface CacheEntryPredicate extends IgnitePredicate, * @param locked Entry locked */ public void entryLocked(boolean locked); -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java index e41938997daa2..62325323e1d3e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java @@ -97,4 +97,4 @@ public abstract class CacheEntryPredicateAdapter implements CacheEntryPredicate @Override public void onAckReceived() { // No-op. } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java index 76806a44f5cfd..73a98b74ac0cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java @@ -140,4 +140,4 @@ public CacheEntryPredicateContainsValue(CacheObject val) { @Override public String toString() { return S.toString(CacheEntryPredicateContainsValue.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java index cac04357a5816..210cc7059b21d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java @@ -28,4 +28,4 @@ public class CacheEntryPredicateHasValue extends CacheEntryPredicateAdapter { @Override public boolean apply(GridCacheEntryEx e) { return peekVisibleValue(e) != null; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java index 2790170e959e8..4c8917fcc68be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java @@ -28,4 +28,4 @@ public class CacheEntryPredicateNoValue extends CacheEntryPredicateAdapter { @Override public boolean apply(GridCacheEntryEx e) { return peekVisibleValue(e) == null; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java index 9057e41fbfbfc..257433636e6b4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java @@ -156,4 +156,4 @@ public CacheEntryPredicate predicate() { @Override public byte fieldsCount() { return 1; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java index 2717b1e03e22e..96b85df2f0721 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java @@ -185,4 +185,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite @Override public byte fieldsCount() { return 3; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java index 3b463afe8a25e..3f880339eb889 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java @@ -267,4 +267,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite @Override public String toString() { return S.toString(CacheInvokeDirectResult.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java index 3bc2a6dcb0624..f9f384a7f9702 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java @@ -118,4 +118,4 @@ public interface CacheObject extends Message { * @throws IgniteCheckedException If failed. */ public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException; -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java index 67ee410d06e2e..c6d900250f6b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java @@ -247,4 +247,4 @@ else if (off >= headSize) return true; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java index 57a70f83e5044..de5a9191950c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java @@ -191,4 +191,4 @@ public CacheObjectByteArrayImpl(byte[] val) { @Override public String toString() { return "CacheObjectByteArrayImpl [arrLen=" + (val != null ? val.length : 0) + ']'; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java index 2124a97940b9b..b29c19e1e254c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java @@ -152,4 +152,4 @@ else if (kernalCtx.config().isPeerClassLoadingEnabled()) @Override public CacheObject prepareForCache(CacheObjectContext ctx) { return this; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java index cf4344d9b74b1..f811f6db9c927 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java @@ -233,15 +233,25 @@ public List nodesByPartition(int part, AffinityTopologyVersion topV * @return Affinity assignment. */ public AffinityAssignment assignment(AffinityTopologyVersion topVer) { + return assignment(topVer, cctx.shared().exchange().lastAffinityChangedTopologyVersion(topVer)); + } + + /** + * Get affinity assignment for the given topology version. + * + * @param topVer Topology version. + * @return Affinity assignment. + */ + public AffinityAssignment assignment(AffinityTopologyVersion topVer, AffinityTopologyVersion lastAffChangedTopVer) { if (cctx.isLocal()) - topVer = LOC_CACHE_TOP_VER; + topVer = lastAffChangedTopVer = LOC_CACHE_TOP_VER; GridAffinityAssignmentCache aff0 = aff; if (aff0 == null) throw new IgniteException(FAILED_TO_FIND_CACHE_ERR_MSG + cctx.name()); - return aff0.cachedAffinity(topVer); + return aff0.cachedAffinity(topVer, lastAffChangedTopVer); } public MvccCoordinator mvccCoordinator(AffinityTopologyVersion topVer) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java index 09c143b0c0dd3..bfdce35e86e62 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java @@ -50,7 +50,7 @@ public int groupId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 3; + return 4; } /** {@inheritDoc} */ @@ -68,7 +68,7 @@ public int groupId() { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeInt("grpId", grpId)) return false; @@ -90,7 +90,7 @@ public int groupId() { return false; switch (reader.state()) { - case 2: + case 3: grpId = reader.readInt("grpId"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java index 6c20bdd15bdd6..e0944397ecf3d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java @@ -52,7 +52,7 @@ public void cacheId(int cacheId) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 3; + return 4; } /** {@inheritDoc} */ @@ -70,7 +70,7 @@ public void cacheId(int cacheId) { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeInt("cacheId", cacheId)) return false; @@ -92,7 +92,7 @@ public void cacheId(int cacheId) { return false; switch (reader.state()) { - case 2: + case 3: cacheId = reader.readInt("cacheId"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java index 2e66e5bfc3fe4..37ca919eca947 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java @@ -165,6 +165,11 @@ public void dumpPendingMessages(StringBuilder sb) { final GridCacheMessage cacheMsg = (GridCacheMessage)msg; + AffinityTopologyVersion rmtAffVer = cacheMsg.topologyVersion(); + AffinityTopologyVersion lastAffChangedVer = cacheMsg.lastAffinityChangedTopologyVersion(); + + cctx.exchange().lastAffinityChangedTopologyVersion(rmtAffVer, lastAffChangedVer); + IgniteInternalFuture fut = null; if (cacheMsg.partitionExchangeMessage()) { @@ -222,9 +227,8 @@ else if (desc.receivedFromStartVersion() != null) } else { AffinityTopologyVersion locAffVer = cctx.exchange().readyAffinityVersion(); - AffinityTopologyVersion rmtAffVer = cacheMsg.topologyVersion(); - if (locAffVer.compareTo(rmtAffVer) < 0) { + if (locAffVer.compareTo(lastAffChangedVer) < 0) { IgniteLogger log = cacheMsg.messageLogger(cctx); if (log.isDebugEnabled()) { @@ -234,12 +238,13 @@ else if (desc.receivedFromStartVersion() != null) msg0.append(", locTopVer=").append(locAffVer). append(", rmtTopVer=").append(rmtAffVer). + append(", lastAffChangedVer=").append(lastAffChangedVer). append(']'); log.debug(msg0.toString()); } - fut = cctx.exchange().affinityReadyFuture(rmtAffVer); + fut = cctx.exchange().affinityReadyFuture(lastAffChangedVer); } } @@ -1155,6 +1160,8 @@ public boolean checkNodeLeft(UUID nodeId, IgniteCheckedException sndErr, boolean public void send(ClusterNode node, GridCacheMessage msg, byte plc) throws IgniteCheckedException { assert !node.isLocal() : node; + msg.lastAffinityChangedTopologyVersion(cctx.exchange().lastAffinityChangedTopologyVersion(msg.topologyVersion())); + if (!onSend(msg, node.id())) return; @@ -1222,6 +1229,8 @@ public void sendOrderedMessage(ClusterNode node, Object topic, GridCacheMessage if (!onSend(msg, node.id())) return; + msg.lastAffinityChangedTopologyVersion(cctx.exchange().lastAffinityChangedTopologyVersion(msg.topologyVersion())); + int cnt = 0; while (cnt <= retryCnt) { @@ -1278,6 +1287,8 @@ void sendNoRetry(ClusterNode node, if (!onSend(msg, null)) return; + msg.lastAffinityChangedTopologyVersion(cctx.exchange().lastAffinityChangedTopologyVersion(msg.topologyVersion())); + try { cctx.gridIO().sendToGridTopic(node, TOPIC_CACHE, msg, plc); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java index 11916e92270fd..f6c2e2eb93831 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java @@ -68,6 +68,10 @@ public abstract class GridCacheMessage implements Message { @GridToStringInclude private GridDeploymentInfoBean depInfo; + /** */ + @GridToStringInclude + private @Nullable AffinityTopologyVersion lastAffChangedTopVer; + /** */ @GridDirectTransient protected boolean addDepInfo; @@ -184,6 +188,27 @@ public AffinityTopologyVersion topologyVersion() { return AffinityTopologyVersion.NONE; } + /** + * Returns the earliest affinity topology version for which this message is valid. + * + * @return Last affinity topology version when affinity was modified. + */ + public AffinityTopologyVersion lastAffinityChangedTopologyVersion() { + if (lastAffChangedTopVer == null || lastAffChangedTopVer.topologyVersion() <= 0) + return topologyVersion(); + + return lastAffChangedTopVer; + } + + /** + * Sets the earliest affinity topology version for which this message is valid. + * + * @param topVer Last affinity topology version when affinity was modified. + */ + public void lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer) { + lastAffChangedTopVer = topVer; + } + /** * Deployment enabled flag indicates whether deployment info has to be added to this message. * @@ -637,7 +662,7 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 2; + return 3; } /** {@inheritDoc} */ @@ -659,6 +684,12 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) { writer.incrementState(); case 1: + if (!writer.writeAffinityTopologyVersion("lastAffChangedTopVer", lastAffChangedTopVer)) + return false; + + writer.incrementState(); + + case 2: if (!writer.writeLong("msgId", msgId)) return false; @@ -686,6 +717,14 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) { reader.incrementState(); case 1: + lastAffChangedTopVer = reader.readAffinityTopologyVersion("lastAffChangedTopVer"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 2: msgId = reader.readLong("msgId"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index eb8901065381e..f11e7843abb01 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +34,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; @@ -133,6 +133,7 @@ import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE; import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; +import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.nextDumpTimeout; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.DFLT_PRELOAD_RESEND_TIMEOUT; @@ -175,19 +176,23 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana @Nullable private volatile GridDhtPartitionsExchangeFuture lastInitializedFut; /** */ - private final AtomicReference lastFinishedFut = new AtomicReference<>(); + private final AtomicReference lastFinishedFut = new AtomicReference<>(); /** */ private final ConcurrentMap readyFuts = new ConcurrentSkipListMap<>(); + /** */ + private final ConcurrentNavigableMap lastAffTopVers = + new ConcurrentSkipListMap<>(); + /** * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE} * means that previous rebalance is undefined and the new one should be initiated. * * Should not be used to determine latest rebalanced topology. */ - private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE; + private volatile AffinityTopologyVersion rebTopVer = NONE; /** */ private GridFutureAdapter reconnectExchangeFut; @@ -893,11 +898,11 @@ public GridDhtPartitionsExchangeFuture lastTopologyFuture() { /** * @param fut Finished future. */ - public void lastFinishedFuture(GridDhtTopologyFuture fut) { + public void lastFinishedFuture(GridDhtPartitionsExchangeFuture fut) { assert fut != null && fut.isDone() : fut; while (true) { - GridDhtTopologyFuture cur = lastFinishedFut.get(); + GridDhtPartitionsExchangeFuture cur = lastFinishedFut.get(); if (fut.topologyVersion() != null && (cur == null || fut.topologyVersion().compareTo(cur.topologyVersion()) > 0)) { if (lastFinishedFut.compareAndSet(cur, fut)) @@ -913,6 +918,17 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) { * @return Future or {@code null} is future is already completed. */ @Nullable public IgniteInternalFuture affinityReadyFuture(AffinityTopologyVersion ver) { + GridDhtPartitionsExchangeFuture lastInitializedFut0 = lastInitializedFut; + + if (lastInitializedFut0 != null && lastInitializedFut0.initialVersion().compareTo(ver) == 0 + && lastInitializedFut0.changedAffinity()) { + if (log.isTraceEnabled()) + log.trace("Return lastInitializedFut for topology ready future " + + "[ver=" + ver + ", fut=" + lastInitializedFut0 + ']'); + + return lastInitializedFut0; + } + AffinityTopologyVersion topVer = exchFuts.readyTopVer(); if (topVer.compareTo(ver) >= 0) { @@ -977,6 +993,48 @@ public boolean hasPendingExchange() { return exchWorker.hasPendingExchange(); } + /** + * + * @param topVer Topology version. + * @return Last topology version before the provided one when affinity was modified. + */ + public AffinityTopologyVersion lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer) { + if (topVer.topologyVersion() <= 0) + return topVer; + + AffinityTopologyVersion lastAffTopVer = lastAffTopVers.get(topVer); + + return lastAffTopVer != null ? lastAffTopVer : topVer; + } + + /** + * + * @param topVer Topology version. + * @param lastAffTopVer Last topology version before the provided one when affinity was modified. + * @return {@code True} if data was modified. + */ + public boolean lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer, AffinityTopologyVersion lastAffTopVer) { + assert lastAffTopVer.compareTo(topVer) <= 0; + + if (lastAffTopVer.topologyVersion() <= 0 || lastAffTopVer.equals(topVer)) + return false; + + while (true) { + AffinityTopologyVersion old = lastAffTopVers.putIfAbsent(topVer, lastAffTopVer); + + if (old == null) + return true; + + if (lastAffTopVer.compareTo(old) < 0) { + if (lastAffTopVers.replace(topVer, old, lastAffTopVer)) + return true; + } + else + return false; + } + + } + /** * @param evt Discovery event. * @return Affinity topology version. @@ -1040,7 +1098,7 @@ public void refreshPartitions() { return; } - ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE); + ClusterNode oldest = cctx.discovery().oldestAliveServerNode(NONE); if (oldest == null) { if (log.isDebugEnabled()) @@ -1463,30 +1521,10 @@ public void onExchangeDone(AffinityTopologyVersion topVer, AffinityTopologyVersi if (log.isDebugEnabled()) log.debug("Exchange done [topVer=" + topVer + ", err=" + err + ']'); - if (err == null) { + if (err == null) exchFuts.readyTopVer(topVer); - for (Map.Entry entry : readyFuts.entrySet()) { - if (entry.getKey().compareTo(topVer) <= 0) { - if (log.isDebugEnabled()) - log.debug("Completing created topology ready future " + - "[ver=" + topVer + ", fut=" + entry.getValue() + ']'); - - entry.getValue().onDone(topVer); - } - } - } - else { - for (Map.Entry entry : readyFuts.entrySet()) { - if (entry.getKey().compareTo(initTopVer) <= 0) { - if (log.isDebugEnabled()) - log.debug("Completing created topology ready future with error " + - "[ver=" + entry.getKey() + ", fut=" + entry.getValue() + ']'); - - entry.getValue().onDone(err); - } - } - } + completeAffReadyFuts(err == null ? topVer : initTopVer, err); ExchangeFutureSet exchFuts0 = exchFuts; @@ -1505,6 +1543,28 @@ public void onExchangeDone(AffinityTopologyVersion topVer, AffinityTopologyVersi } } + /** */ + private void completeAffReadyFuts(AffinityTopologyVersion topVer, @Nullable Throwable err) { + for (Map.Entry entry : readyFuts.entrySet()) { + if (entry.getKey().compareTo(topVer) <= 0) { + if (err == null) { + if (log.isDebugEnabled()) + log.debug("Completing created topology ready future " + + "[ver=" + topVer + ", fut=" + entry.getValue() + ']'); + + entry.getValue().onDone(topVer); + } + else { + if (log.isDebugEnabled()) + log.debug("Completing created topology ready future with error " + + "[ver=" + entry.getKey() + ", fut=" + entry.getValue() + ']'); + + entry.getValue().onDone(err); + } + } + } + } + /** * @param fut Future. * @return {@code True} if added. @@ -2139,6 +2199,13 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu break; } + if (!fut.changedAffinity()) { + if (log.isInfoEnabled()) + log.info("Stop merge, no-affinity exchange found: " + evt); + + break; + } + ClusterNode node = evt.eventNode(); if ((evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT) && @@ -2311,6 +2378,38 @@ private boolean exchangeInProgress() { return false; } + /** */ + public boolean affinityChanged(AffinityTopologyVersion from, AffinityTopologyVersion to) { + if (lastAffinityChangedTopologyVersion(to).compareTo(from) >= 0) + return false; + + Collection history = exchFuts.values(); + + boolean fromFound = false; + + for (GridDhtPartitionsExchangeFuture fut : history) { + if (!fromFound) { + int cmp = fut.initialVersion().compareTo(from); + + if (cmp > 0) // We don't have history, so return true for safety + return true; + else if (cmp == 0) + fromFound = true; + else if (fut.isDone() && fut.topologyVersion().compareTo(from) >= 0) + return true; // Temporary solution for merge exchange case + } + else { + if (fut.changedAffinity()) + return true; + + if (fut.initialVersion().compareTo(to) >= 0) + return false; + } + } + + return true; + } + /** * Exchange future thread. All exchanges happen only by one thread and next * exchange will not start until previous one completes. @@ -2664,6 +2763,20 @@ else if (task instanceof ForceRebalanceExchangeTask) { crd = newCrd = !srvNodes.isEmpty() && srvNodes.get(0).isLocal(); } + if (!exchFut.changedAffinity()) { + GridDhtPartitionsExchangeFuture lastFut = lastFinishedFut.get(); + + if (lastFut != null) { + if (!lastFut.changedAffinity()) { + AffinityTopologyVersion lastAffVer = cctx.exchange().lastAffinityChangedTopologyVersion(lastFut.initialVersion()); + + cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastAffVer); + } + else + cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastFut.initialVersion()); + } + } + exchFut.init(newCrd); int dumpCnt = 0; @@ -2757,7 +2870,7 @@ else if (task instanceof ForceRebalanceExchangeTask) { continue; if (grp.preloader().rebalanceRequired(rebTopVer, exchFut)) - rebTopVer = AffinityTopologyVersion.NONE; + rebTopVer = NONE; changed |= grp.topology().afterExchange(exchFut); } @@ -2768,9 +2881,9 @@ else if (task instanceof ForceRebalanceExchangeTask) { // Schedule rebalance if force rebalance or force reassign occurs. if (exchFut == null) - rebTopVer = AffinityTopologyVersion.NONE; + rebTopVer = NONE; - if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) { + if (!cctx.kernalContext().clientNode() && rebTopVer.equals(NONE)) { assignsMap = new HashMap<>(); IgniteCacheSnapshotManager snp = cctx.snapshot(); @@ -2801,7 +2914,7 @@ else if (task instanceof ForceRebalanceExchangeTask) { busy = false; } - if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) { + if (assignsMap != null && rebTopVer.equals(NONE)) { int size = assignsMap.size(); NavigableMap> orderMap = new TreeMap<>(); @@ -2984,7 +3097,7 @@ private static class ExchangeFutureSet extends GridListSet readyTopVer = - new AtomicReference<>(AffinityTopologyVersion.NONE); + new AtomicReference<>(NONE); /** * Creates ordered, not strict list set. @@ -2992,20 +3105,15 @@ private static class ExchangeFutureSet extends GridListSet() { - @Override public int compare( - GridDhtPartitionsExchangeFuture f1, - GridDhtPartitionsExchangeFuture f2 - ) { - AffinityTopologyVersion t1 = f1.exchangeId().topologyVersion(); - AffinityTopologyVersion t2 = f2.exchangeId().topologyVersion(); - - assert t1.topologyVersion() > 0; - assert t2.topologyVersion() > 0; - - // Reverse order. - return t2.compareTo(t1); - } + super((f1, f2) -> { + AffinityTopologyVersion t1 = f1.exchangeId().topologyVersion(); + AffinityTopologyVersion t2 = f2.exchangeId().topologyVersion(); + + assert t1.topologyVersion() > 0; + assert t2.topologyVersion() > 0; + + // Reverse order. + return t2.compareTo(t1); }, /*not strict*/false); this.histSize = histSize; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java index e49be4934eeb9..4cf9e2388229c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java @@ -116,13 +116,13 @@ public Throwable getError() { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 3: + case 4: if (!writer.writeUuid("requestId", requestId)) return false; @@ -144,7 +144,7 @@ public Throwable getError() { return false; switch (reader.state()) { - case 2: + case 3: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -152,7 +152,7 @@ public Throwable getError() { reader.incrementState(); - case 3: + case 4: requestId = reader.readUuid("requestId"); if (!reader.isLastRead()) @@ -172,7 +172,7 @@ public Throwable getError() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 4; + return 5; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java index 8f8ceb6d256f1..9c4eeee6cceda 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java @@ -48,4 +48,4 @@ public interface KeyCacheObject extends CacheObject { * @return Copy of this object with given partition set. */ public KeyCacheObject copy(int part); -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java index 7c241068231bb..f8012a7de733e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.query.schema.message.SchemaOperationStatusMessage; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; @@ -31,6 +32,7 @@ /** * WAL state ack message (sent from participant node to coordinator). */ +@IgniteCodeGeneratingFail public class WalStateAckMessage implements Message { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java index c092132192c74..c420aeb4f0491 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java @@ -213,37 +213,37 @@ public List nearVersions() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeCollection("nearVers", nearVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: - if (!writer.writeMessage("topVer", topVer)) + case 7: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeLong("ttl", ttl)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeCollection("vers", vers, MessageCollectionItemType.MSG)) return false; @@ -265,7 +265,7 @@ public List nearVersions() { return false; switch (reader.state()) { - case 3: + case 4: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -273,7 +273,7 @@ public List nearVersions() { reader.incrementState(); - case 4: + case 5: nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -281,7 +281,7 @@ public List nearVersions() { reader.incrementState(); - case 5: + case 6: nearVers = reader.readCollection("nearVers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -289,15 +289,15 @@ public List nearVersions() { reader.incrementState(); - case 6: - topVer = reader.readMessage("topVer"); + case 7: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 7: + case 8: ttl = reader.readLong("ttl"); if (!reader.isLastRead()) @@ -305,7 +305,7 @@ public List nearVersions() { reader.incrementState(); - case 8: + case 9: vers = reader.readCollection("vers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -325,7 +325,7 @@ public List nearVersions() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 9; + return 10; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java index 45d1f1a31bf5a..90ce2344d77ab 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java @@ -148,37 +148,37 @@ public boolean system() { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeBoolean("nearTxCheck", nearTxCheck)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeMessage("nearXidVer", nearXidVer)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeBoolean("sys", sys)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeInt("txNum", txNum)) return false; @@ -200,7 +200,7 @@ public boolean system() { return false; switch (reader.state()) { - case 7: + case 8: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -208,7 +208,7 @@ public boolean system() { reader.incrementState(); - case 8: + case 9: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -216,7 +216,7 @@ public boolean system() { reader.incrementState(); - case 9: + case 10: nearTxCheck = reader.readBoolean("nearTxCheck"); if (!reader.isLastRead()) @@ -224,7 +224,7 @@ public boolean system() { reader.incrementState(); - case 10: + case 11: nearXidVer = reader.readMessage("nearXidVer"); if (!reader.isLastRead()) @@ -232,7 +232,7 @@ public boolean system() { reader.incrementState(); - case 11: + case 12: sys = reader.readBoolean("sys"); if (!reader.isLastRead()) @@ -240,7 +240,7 @@ public boolean system() { reader.incrementState(); - case 12: + case 13: txNum = reader.readInt("txNum"); if (!reader.isLastRead()) @@ -260,7 +260,7 @@ public boolean system() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 13; + return 14; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java index a9ac26ba4b49c..1ef44a8f21f6b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java @@ -129,19 +129,19 @@ public boolean success() { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeBoolean("success", success)) return false; @@ -163,7 +163,7 @@ public boolean success() { return false; switch (reader.state()) { - case 7: + case 8: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -171,7 +171,7 @@ public boolean success() { reader.incrementState(); - case 8: + case 9: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -179,7 +179,7 @@ public boolean success() { reader.incrementState(); - case 9: + case 10: success = reader.readBoolean("success"); if (!reader.isLastRead()) @@ -199,7 +199,7 @@ public boolean success() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java index fc209aaa956f0..8536e480489b5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java @@ -161,25 +161,25 @@ int keysCount() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByteArray("candsByIdxBytes", candsByIdxBytes)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeCollection("committedVers", committedVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeCollection("rolledbackVers", rolledbackVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeMessage("ver", ver)) return false; @@ -201,7 +201,7 @@ int keysCount() { return false; switch (reader.state()) { - case 3: + case 4: candsByIdxBytes = reader.readByteArray("candsByIdxBytes"); if (!reader.isLastRead()) @@ -209,7 +209,7 @@ int keysCount() { reader.incrementState(); - case 4: + case 5: committedVers = reader.readCollection("committedVers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -217,7 +217,7 @@ int keysCount() { reader.incrementState(); - case 5: + case 6: rolledbackVers = reader.readCollection("rolledbackVers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -225,7 +225,7 @@ int keysCount() { reader.incrementState(); - case 6: + case 7: ver = reader.readMessage("ver"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java index 25a557c324817..ca78763fc2148 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java @@ -366,79 +366,79 @@ public long timeout() { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeBoolean("isInTx", isInTx)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeBoolean("isInvalidate", isInvalidate)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeBoolean("isRead", isRead)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeMessage("nearXidVer", nearXidVer)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeUuid("nodeId", nodeId)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeBooleanArray("retVals", retVals)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeLong("threadId", threadId)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 19: + case 20: if (!writer.writeInt("txSize", txSize)) return false; @@ -460,7 +460,7 @@ public long timeout() { return false; switch (reader.state()) { - case 7: + case 8: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -468,7 +468,7 @@ public long timeout() { reader.incrementState(); - case 8: + case 9: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -476,7 +476,7 @@ public long timeout() { reader.incrementState(); - case 9: + case 10: isInTx = reader.readBoolean("isInTx"); if (!reader.isLastRead()) @@ -484,7 +484,7 @@ public long timeout() { reader.incrementState(); - case 10: + case 11: isInvalidate = reader.readBoolean("isInvalidate"); if (!reader.isLastRead()) @@ -492,7 +492,7 @@ public long timeout() { reader.incrementState(); - case 11: + case 12: isRead = reader.readBoolean("isRead"); if (!reader.isLastRead()) @@ -500,7 +500,7 @@ public long timeout() { reader.incrementState(); - case 12: + case 13: byte isolationOrd; isolationOrd = reader.readByte("isolation"); @@ -512,7 +512,7 @@ public long timeout() { reader.incrementState(); - case 13: + case 14: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -520,7 +520,7 @@ public long timeout() { reader.incrementState(); - case 14: + case 15: nearXidVer = reader.readMessage("nearXidVer"); if (!reader.isLastRead()) @@ -528,7 +528,7 @@ public long timeout() { reader.incrementState(); - case 15: + case 16: nodeId = reader.readUuid("nodeId"); if (!reader.isLastRead()) @@ -536,7 +536,7 @@ public long timeout() { reader.incrementState(); - case 16: + case 17: retVals = reader.readBooleanArray("retVals"); if (!reader.isLastRead()) @@ -544,7 +544,7 @@ public long timeout() { reader.incrementState(); - case 17: + case 18: threadId = reader.readLong("threadId"); if (!reader.isLastRead()) @@ -552,7 +552,7 @@ public long timeout() { reader.incrementState(); - case 18: + case 19: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -560,7 +560,7 @@ public long timeout() { reader.incrementState(); - case 19: + case 20: txSize = reader.readInt("txSize"); if (!reader.isLastRead()) @@ -580,7 +580,7 @@ public long timeout() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 20; + return 21; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java index 4b21896b1c051..2d4de9c8156eb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java @@ -221,19 +221,19 @@ protected int valuesSize() { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG)) return false; @@ -255,7 +255,7 @@ protected int valuesSize() { return false; switch (reader.state()) { - case 7: + case 8: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -263,7 +263,7 @@ protected int valuesSize() { reader.incrementState(); - case 8: + case 9: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -271,7 +271,7 @@ protected int valuesSize() { reader.incrementState(); - case 9: + case 10: vals = reader.readCollection("vals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -291,7 +291,7 @@ protected int valuesSize() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java index ea9336b2a55e3..a1af470c56deb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java @@ -325,85 +325,85 @@ public boolean replyRequired() { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeMessage("baseVer", baseVer)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeBoolean("commit", commit)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeMessage("commitVer", commitVer)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeBoolean("invalidate", invalidate)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeByte("plc", plc)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeBoolean("sys", sys)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeLong("threadId", threadId)) return false; writer.incrementState(); - case 19: - if (!writer.writeMessage("topVer", topVer)) + case 20: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 20: + case 21: if (!writer.writeInt("txSize", txSize)) return false; @@ -425,7 +425,7 @@ public boolean replyRequired() { return false; switch (reader.state()) { - case 7: + case 8: baseVer = reader.readMessage("baseVer"); if (!reader.isLastRead()) @@ -433,7 +433,7 @@ public boolean replyRequired() { reader.incrementState(); - case 8: + case 9: commit = reader.readBoolean("commit"); if (!reader.isLastRead()) @@ -441,7 +441,7 @@ public boolean replyRequired() { reader.incrementState(); - case 9: + case 10: commitVer = reader.readMessage("commitVer"); if (!reader.isLastRead()) @@ -449,7 +449,7 @@ public boolean replyRequired() { reader.incrementState(); - case 10: + case 11: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -457,7 +457,7 @@ public boolean replyRequired() { reader.incrementState(); - case 11: + case 12: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -465,7 +465,7 @@ public boolean replyRequired() { reader.incrementState(); - case 12: + case 13: invalidate = reader.readBoolean("invalidate"); if (!reader.isLastRead()) @@ -473,7 +473,7 @@ public boolean replyRequired() { reader.incrementState(); - case 13: + case 14: plc = reader.readByte("plc"); if (!reader.isLastRead()) @@ -481,7 +481,7 @@ public boolean replyRequired() { reader.incrementState(); - case 14: + case 15: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -489,7 +489,7 @@ public boolean replyRequired() { reader.incrementState(); - case 15: + case 16: byte syncModeOrd; syncModeOrd = reader.readByte("syncMode"); @@ -501,7 +501,7 @@ public boolean replyRequired() { reader.incrementState(); - case 16: + case 17: sys = reader.readBoolean("sys"); if (!reader.isLastRead()) @@ -509,7 +509,7 @@ public boolean replyRequired() { reader.incrementState(); - case 17: + case 18: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -517,7 +517,7 @@ public boolean replyRequired() { reader.incrementState(); - case 18: + case 19: threadId = reader.readLong("threadId"); if (!reader.isLastRead()) @@ -525,15 +525,15 @@ public boolean replyRequired() { reader.incrementState(); - case 19: - topVer = reader.readMessage("topVer"); + case 20: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 20: + case 21: txSize = reader.readInt("txSize"); if (!reader.isLastRead()) @@ -553,7 +553,7 @@ public boolean replyRequired() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 21; + return 22; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java index c36e6336d4f7c..5fdf970bc0f8d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java @@ -145,25 +145,25 @@ public IgniteUuid futureId() { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 3: + case 4: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeInt("part", part)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeMessage("txId", txId)) return false; @@ -185,7 +185,7 @@ public IgniteUuid futureId() { return false; switch (reader.state()) { - case 2: + case 3: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -193,7 +193,7 @@ public IgniteUuid futureId() { reader.incrementState(); - case 3: + case 4: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -201,7 +201,7 @@ public IgniteUuid futureId() { reader.incrementState(); - case 4: + case 5: part = reader.readInt("part"); if (!reader.isLastRead()) @@ -209,7 +209,7 @@ public IgniteUuid futureId() { reader.incrementState(); - case 5: + case 6: txId = reader.readMessage("txId"); if (!reader.isLastRead()) @@ -229,7 +229,7 @@ public IgniteUuid futureId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 6; + return 7; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java index a5aa0d838935a..96eeee20b4b71 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java @@ -505,79 +505,79 @@ private boolean isFlag(int mask) { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeByte("concurrency", concurrency != null ? (byte)concurrency.ordinal() : -1)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeCollection("dhtVerKeys", dhtVerKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeCollection("dhtVerVals", dhtVerVals, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeByte("plc", plc)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeCollection("reads", reads, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeLong("threadId", threadId)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeMap("txNodesMsg", txNodesMsg, MessageCollectionItemType.UUID, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeInt("txSize", txSize)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeMessage("writeVer", writeVer)) return false; writer.incrementState(); - case 19: + case 20: if (!writer.writeCollection("writes", writes, MessageCollectionItemType.MSG)) return false; @@ -599,7 +599,7 @@ private boolean isFlag(int mask) { return false; switch (reader.state()) { - case 7: + case 8: byte concurrencyOrd; concurrencyOrd = reader.readByte("concurrency"); @@ -611,7 +611,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 8: + case 9: dhtVerKeys = reader.readCollection("dhtVerKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -619,7 +619,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 9: + case 10: dhtVerVals = reader.readCollection("dhtVerVals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -627,7 +627,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 10: + case 11: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -635,7 +635,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 11: + case 12: byte isolationOrd; isolationOrd = reader.readByte("isolation"); @@ -647,7 +647,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 12: + case 13: plc = reader.readByte("plc"); if (!reader.isLastRead()) @@ -655,7 +655,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 13: + case 14: reads = reader.readCollection("reads", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -663,7 +663,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 14: + case 15: threadId = reader.readLong("threadId"); if (!reader.isLastRead()) @@ -671,7 +671,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 15: + case 16: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -679,7 +679,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 16: + case 17: txNodesMsg = reader.readMap("txNodesMsg", MessageCollectionItemType.UUID, MessageCollectionItemType.MSG, false); if (!reader.isLastRead()) @@ -687,7 +687,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 17: + case 18: txSize = reader.readInt("txSize"); if (!reader.isLastRead()) @@ -695,7 +695,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 18: + case 19: writeVer = reader.readMessage("writeVer"); if (!reader.isLastRead()) @@ -703,7 +703,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 19: + case 20: writes = reader.readCollection("writes", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -723,7 +723,7 @@ private boolean isFlag(int mask) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 20; + return 21; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java index 58e94926ca9e8..c26880e10fed4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java @@ -178,19 +178,19 @@ public boolean isRollback() { } switch (writer.state()) { - case 7: + case 8: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeInt("part", part)) return false; @@ -212,7 +212,7 @@ public boolean isRollback() { return false; switch (reader.state()) { - case 7: + case 8: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -220,7 +220,7 @@ public boolean isRollback() { reader.incrementState(); - case 8: + case 9: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -228,7 +228,7 @@ public boolean isRollback() { reader.incrementState(); - case 9: + case 10: part = reader.readInt("part"); if (!reader.isLastRead()) @@ -248,7 +248,7 @@ public boolean isRollback() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java index ca2bdab156bf7..001eb61589569 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java @@ -122,7 +122,7 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke } switch (writer.state()) { - case 7: + case 8: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; @@ -144,7 +144,7 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke return false; switch (reader.state()) { - case 7: + case 8: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -164,7 +164,7 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 8; + return 9; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java index 4b48f5a59e9ff..f468590a2c198 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java @@ -80,6 +80,11 @@ public void validate(CacheGroupContext grp, Collection topNodes) { return topVer; } + /** {@inheritDoc} */ + @Override public boolean changedAffinity() { + return true; + } + /** {@inheritDoc} */ @Override public String toString() { return "ClientCacheDhtTopologyFuture [topVer=" + topVer + ']'; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java index 44c7b88c029f3..cf7018a15c895 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java @@ -109,7 +109,7 @@ public long futureId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 6; + return 7; } /** {@inheritDoc} */ @@ -127,20 +127,20 @@ public long futureId() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 5: - if (!writer.writeMessage("topVer", topVer)) + case 6: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -161,7 +161,7 @@ public long futureId() { return false; switch (reader.state()) { - case 3: + case 4: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -169,7 +169,7 @@ public long futureId() { reader.incrementState(); - case 4: + case 5: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -177,8 +177,8 @@ public long futureId() { reader.incrementState(); - case 5: - topVer = reader.readMessage("topVer"); + case 6: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java index 5b0de08a2ce97..e8b40e9cd8c15 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java @@ -215,7 +215,7 @@ private List> ids(List> assignments) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 8; + return 9; } /** @@ -272,32 +272,32 @@ private List> ids(List> assignments) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByteArray("affAssignmentIdsBytes", affAssignmentIdsBytes)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeByteArray("idealAffAssignmentBytes", idealAffAssignmentBytes)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeByteArray("partBytes", partBytes)) return false; writer.incrementState(); - case 7: - if (!writer.writeMessage("topVer", topVer)) + case 8: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -318,7 +318,7 @@ private List> ids(List> assignments) { return false; switch (reader.state()) { - case 3: + case 4: affAssignmentIdsBytes = reader.readByteArray("affAssignmentIdsBytes"); if (!reader.isLastRead()) @@ -326,7 +326,7 @@ private List> ids(List> assignments) { reader.incrementState(); - case 4: + case 5: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -334,7 +334,7 @@ private List> ids(List> assignments) { reader.incrementState(); - case 5: + case 6: idealAffAssignmentBytes = reader.readByteArray("idealAffAssignmentBytes"); if (!reader.isLastRead()) @@ -342,7 +342,7 @@ private List> ids(List> assignments) { reader.incrementState(); - case 6: + case 7: partBytes = reader.readByteArray("partBytes"); if (!reader.isLastRead()) @@ -350,8 +350,8 @@ private List> ids(List> assignments) { reader.incrementState(); - case 7: - topVer = reader.readMessage("topVer"); + case 8: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java index 71ddf3ce189bc..f69f8132410ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java @@ -1263,6 +1263,11 @@ protected final boolean needRemap(AffinityTopologyVersion expVer, AffinityTopolo if (expVer.equals(curVer)) return false; + AffinityTopologyVersion lastAffChangedTopVer = ctx.shared().exchange().lastAffinityChangedTopologyVersion(expVer); + + if (curVer.compareTo(lastAffChangedTopVer) >= 0 && curVer.compareTo(expVer) <= 0) + return false; + // TODO IGNITE-7164 check mvcc crd for mvcc enabled txs. Collection cacheNodes0 = ctx.discovery().cacheGroupAffinityNodes(ctx.groupId(), expVer); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java index 1ac58182ddc57..76c3b163d577a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java @@ -363,62 +363,62 @@ public long accessTtl() { } switch (writer.state()) { - case 20: + case 21: if (!writer.writeLong("accessTtl", accessTtl)) return false; writer.incrementState(); - case 21: + case 22: if (!writer.writeBitSet("invalidateEntries", invalidateEntries)) return false; writer.incrementState(); - case 22: + case 23: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 23: + case 24: if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 24: + case 25: if (!writer.writeObjectArray("ownedKeys", ownedKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 25: + case 26: if (!writer.writeObjectArray("ownedValues", ownedValues, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 26: + case 27: if (!writer.writeBitSet("preloadKeys", preloadKeys)) return false; writer.incrementState(); - case 27: + case 28: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 28: + case 29: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 29: - if (!writer.writeMessage("topVer", topVer)) + case 30: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -439,7 +439,7 @@ public long accessTtl() { return false; switch (reader.state()) { - case 20: + case 21: accessTtl = reader.readLong("accessTtl"); if (!reader.isLastRead()) @@ -447,7 +447,7 @@ public long accessTtl() { reader.incrementState(); - case 21: + case 22: invalidateEntries = reader.readBitSet("invalidateEntries"); if (!reader.isLastRead()) @@ -455,7 +455,7 @@ public long accessTtl() { reader.incrementState(); - case 22: + case 23: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -463,7 +463,7 @@ public long accessTtl() { reader.incrementState(); - case 23: + case 24: nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -471,7 +471,7 @@ public long accessTtl() { reader.incrementState(); - case 24: + case 25: ownedKeys = reader.readObjectArray("ownedKeys", MessageCollectionItemType.MSG, KeyCacheObject.class); if (!reader.isLastRead()) @@ -479,7 +479,7 @@ public long accessTtl() { reader.incrementState(); - case 25: + case 26: ownedValues = reader.readObjectArray("ownedValues", MessageCollectionItemType.MSG, GridCacheVersion.class); if (!reader.isLastRead()) @@ -487,7 +487,7 @@ public long accessTtl() { reader.incrementState(); - case 26: + case 27: preloadKeys = reader.readBitSet("preloadKeys"); if (!reader.isLastRead()) @@ -495,7 +495,7 @@ public long accessTtl() { reader.incrementState(); - case 27: + case 28: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -503,7 +503,7 @@ public long accessTtl() { reader.incrementState(); - case 28: + case 29: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -511,8 +511,8 @@ public long accessTtl() { reader.incrementState(); - case 29: - topVer = reader.readMessage("topVer"); + case 30: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -531,7 +531,7 @@ public long accessTtl() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 30; + return 31; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java index 87abd6c2ce994..63c07e82906f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java @@ -207,25 +207,25 @@ public Collection preloadEntries() { } switch (writer.state()) { - case 10: + case 11: if (!writer.writeCollection("invalidParts", invalidParts, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeCollection("nearEvicted", nearEvicted, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeCollection("preloadEntries", preloadEntries, MessageCollectionItemType.MSG)) return false; @@ -247,7 +247,7 @@ public Collection preloadEntries() { return false; switch (reader.state()) { - case 10: + case 11: invalidParts = reader.readCollection("invalidParts", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -255,7 +255,7 @@ public Collection preloadEntries() { reader.incrementState(); - case 11: + case 12: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -263,7 +263,7 @@ public Collection preloadEntries() { reader.incrementState(); - case 12: + case 13: nearEvicted = reader.readCollection("nearEvicted", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -271,7 +271,7 @@ public Collection preloadEntries() { reader.incrementState(); - case 13: + case 14: preloadEntries = reader.readCollection("preloadEntries", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -291,7 +291,7 @@ public Collection preloadEntries() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 14; + return 15; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java index 489fb63309ea5..3cae875f5d9c7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java @@ -86,4 +86,10 @@ public interface GridDhtTopologyFuture extends IgniteInternalFuture keys); + + /** + * + * @return {@code True} if this exchange changed affinity. + */ + public boolean changedAffinity(); } \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index edd28b44a47bc..66ba325dd6efa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -2124,7 +2124,8 @@ public GridDhtTxLocal initTxTopologyVersion(UUID nodeId, GridDhtTopologyFuture topFut = top.topologyVersionFuture(); - if (!topFut.isDone() || !topFut.topologyVersion().equals(topVer)) { + if (!topFut.isDone() || !(topFut.topologyVersion().compareTo(topVer) >= 0 + && ctx.shared().exchange().lastAffinityChangedTopologyVersion(topFut.initialVersion()).compareTo(topVer) <= 0)) { // TODO IGNITE-7164 Wait for topology change, remap client TX in case affinity was changed. top.readUnlock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java index 61896b59a3d15..04f411d8e3b23 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java @@ -393,44 +393,38 @@ public Collection updateCounters() { } switch (writer.state()) { - case 21: - if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1)) - return false; - - writer.incrementState(); - case 22: - if (!writer.writeInt("miniId", miniId)) + if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1)) return false; writer.incrementState(); case 23: - if (!writer.writeUuid("nearNodeId", nearNodeId)) + if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); case 24: - if (!writer.writeMessage("partUpdateCnt", partUpdateCnt)) + if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); case 25: - if (!writer.writeCollection("pendingVers", pendingVers, MessageCollectionItemType.MSG)) + if (!writer.writeUuid("nearNodeId", nearNodeId)) return false; writer.incrementState(); case 26: - if (!writer.writeMessage("writeVer", writeVer)) + if (!writer.writeMessage("partUpdateCnt", partUpdateCnt)) return false; writer.incrementState(); case 27: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) + if (!writer.writeCollection("pendingVers", pendingVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); @@ -441,6 +435,12 @@ public Collection updateCounters() { writer.incrementState(); + case 29: + if (!writer.writeMessage("writeVer", writeVer)) + return false; + + writer.incrementState(); + } return true; @@ -457,7 +457,7 @@ public Collection updateCounters() { return false; switch (reader.state()) { - case 21: + case 22: byte isolationOrd; isolationOrd = reader.readByte("isolation"); @@ -469,16 +469,8 @@ public Collection updateCounters() { reader.incrementState(); - case 22: - miniId = reader.readInt("miniId"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - case 23: - nearNodeId = reader.readUuid("nearNodeId"); + miniId = reader.readInt("miniId"); if (!reader.isLastRead()) return false; @@ -486,7 +478,7 @@ public Collection updateCounters() { reader.incrementState(); case 24: - partUpdateCnt = reader.readMessage("partUpdateCnt"); + mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) return false; @@ -494,7 +486,7 @@ public Collection updateCounters() { reader.incrementState(); case 25: - pendingVers = reader.readCollection("pendingVers", MessageCollectionItemType.MSG); + nearNodeId = reader.readUuid("nearNodeId"); if (!reader.isLastRead()) return false; @@ -502,7 +494,7 @@ public Collection updateCounters() { reader.incrementState(); case 26: - writeVer = reader.readMessage("writeVer"); + partUpdateCnt = reader.readMessage("partUpdateCnt"); if (!reader.isLastRead()) return false; @@ -510,7 +502,7 @@ public Collection updateCounters() { reader.incrementState(); case 27: - mvccSnapshot = reader.readMessage("mvccSnapshot"); + pendingVers = reader.readCollection("pendingVers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -525,6 +517,14 @@ public Collection updateCounters() { reader.incrementState(); + case 29: + writeVer = reader.readMessage("writeVer"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + } return reader.afterMessageRead(GridDhtTxFinishRequest.class); @@ -537,7 +537,7 @@ public Collection updateCounters() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 29; + return 30; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java index 6d717ebf904e6..d777a2201a149 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java @@ -173,19 +173,19 @@ public GridCacheReturn returnValue() { } switch (writer.state()) { - case 6: + case 7: if (!writer.writeByteArray("checkCommittedErrBytes", checkCommittedErrBytes)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeMessage("retVal", retVal)) return false; @@ -207,7 +207,7 @@ public GridCacheReturn returnValue() { return false; switch (reader.state()) { - case 6: + case 7: checkCommittedErrBytes = reader.readByteArray("checkCommittedErrBytes"); if (!reader.isLastRead()) @@ -215,7 +215,7 @@ public GridCacheReturn returnValue() { reader.incrementState(); - case 7: + case 8: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -223,7 +223,7 @@ public GridCacheReturn returnValue() { reader.incrementState(); - case 8: + case 9: retVal = reader.readMessage("retVal"); if (!reader.isLastRead()) @@ -243,7 +243,7 @@ public GridCacheReturn returnValue() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 9; + return 10; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java index 67eacd3f8c5f8..50f2e7947b1c4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java @@ -97,7 +97,7 @@ public Collection versions() { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeCollection("vers", vers, MessageCollectionItemType.MSG)) return false; @@ -119,7 +119,7 @@ public Collection versions() { return false; switch (reader.state()) { - case 2: + case 3: vers = reader.readCollection("vers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -139,6 +139,6 @@ public Collection versions() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 3; + return 4; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java index 30e8cebbd3638..46a06d781b576 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java @@ -90,7 +90,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest { /** */ @GridDirectCollection(PartitionUpdateCountersMessage.class) - private Collection counters; + private Collection updCntrs; /** Near transaction ID. */ private GridCacheVersion nearXidVer; @@ -136,7 +136,7 @@ public GridDhtTxPrepareRequest() { * @param storeWriteThrough Cache store write through flag. * @param retVal Need return value flag * @param mvccSnapshot Mvcc snapshot. - * @param counters Update counters for mvcc Tx. + * @param updCntrs Update counters for mvcc Tx. */ public GridDhtTxPrepareRequest( IgniteUuid futId, @@ -156,7 +156,7 @@ public GridDhtTxPrepareRequest( boolean storeWriteThrough, boolean retVal, MvccSnapshot mvccSnapshot, - Collection counters) { + Collection updCntrs) { super(tx, timeout, null, @@ -178,7 +178,7 @@ public GridDhtTxPrepareRequest( this.subjId = subjId; this.taskNameHash = taskNameHash; this.mvccSnapshot = mvccSnapshot; - this.counters = counters; + this.updCntrs = updCntrs; storeWriteThrough(storeWriteThrough); needReturnValue(retVal); @@ -201,7 +201,7 @@ public MvccSnapshot mvccSnapshot() { * @return Update counters list. */ public Collection updateCounters() { - return counters; + return updCntrs; } /** @@ -423,92 +423,92 @@ public boolean skipCompletedVersion() { } switch (writer.state()) { - case 20: - if (!writer.writeIgniteUuid("futId", futId)) - return false; - - writer.incrementState(); - case 21: - if (!writer.writeBitSet("invalidateNearEntries", invalidateNearEntries)) + if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); case 22: - if (!writer.writeInt("miniId", miniId)) + if (!writer.writeBitSet("invalidateNearEntries", invalidateNearEntries)) return false; writer.incrementState(); case 23: - if (!writer.writeUuid("nearNodeId", nearNodeId)) + if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); case 24: - if (!writer.writeCollection("nearWrites", nearWrites, MessageCollectionItemType.MSG)) + if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); case 25: - if (!writer.writeMessage("nearXidVer", nearXidVer)) + if (!writer.writeUuid("nearNodeId", nearNodeId)) return false; writer.incrementState(); case 26: - if (!writer.writeCollection("ownedKeys", ownedKeys, MessageCollectionItemType.MSG)) + if (!writer.writeCollection("nearWrites", nearWrites, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 27: - if (!writer.writeCollection("ownedVals", ownedVals, MessageCollectionItemType.MSG)) + if (!writer.writeMessage("nearXidVer", nearXidVer)) return false; writer.incrementState(); case 28: - if (!writer.writeBitSet("preloadKeys", preloadKeys)) + if (!writer.writeCollection("ownedKeys", ownedKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 29: - if (!writer.writeBoolean("skipCompletedVers", skipCompletedVers)) + if (!writer.writeCollection("ownedVals", ownedVals, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 30: - if (!writer.writeUuid("subjId", subjId)) + if (!writer.writeBitSet("preloadKeys", preloadKeys)) return false; writer.incrementState(); case 31: - if (!writer.writeInt("taskNameHash", taskNameHash)) + if (!writer.writeBoolean("skipCompletedVers", skipCompletedVers)) return false; writer.incrementState(); case 32: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); case 33: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) + if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); case 34: - if (!writer.writeCollection("counters", counters, MessageCollectionItemType.MSG)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) + return false; + + writer.incrementState(); + + case 35: + if (!writer.writeCollection("updCntrs", updCntrs, MessageCollectionItemType.MSG)) return false; writer.incrementState(); @@ -529,16 +529,8 @@ public boolean skipCompletedVersion() { return false; switch (reader.state()) { - case 20: - futId = reader.readIgniteUuid("futId"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - case 21: - invalidateNearEntries = reader.readBitSet("invalidateNearEntries"); + futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) return false; @@ -546,7 +538,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 22: - miniId = reader.readInt("miniId"); + invalidateNearEntries = reader.readBitSet("invalidateNearEntries"); if (!reader.isLastRead()) return false; @@ -554,7 +546,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 23: - nearNodeId = reader.readUuid("nearNodeId"); + miniId = reader.readInt("miniId"); if (!reader.isLastRead()) return false; @@ -562,7 +554,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 24: - nearWrites = reader.readCollection("nearWrites", MessageCollectionItemType.MSG); + mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) return false; @@ -570,7 +562,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 25: - nearXidVer = reader.readMessage("nearXidVer"); + nearNodeId = reader.readUuid("nearNodeId"); if (!reader.isLastRead()) return false; @@ -578,7 +570,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 26: - ownedKeys = reader.readCollection("ownedKeys", MessageCollectionItemType.MSG); + nearWrites = reader.readCollection("nearWrites", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -586,7 +578,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 27: - ownedVals = reader.readCollection("ownedVals", MessageCollectionItemType.MSG); + nearXidVer = reader.readMessage("nearXidVer"); if (!reader.isLastRead()) return false; @@ -594,7 +586,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 28: - preloadKeys = reader.readBitSet("preloadKeys"); + ownedKeys = reader.readCollection("ownedKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -602,7 +594,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 29: - skipCompletedVers = reader.readBoolean("skipCompletedVers"); + ownedVals = reader.readCollection("ownedVals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -610,7 +602,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 30: - subjId = reader.readUuid("subjId"); + preloadKeys = reader.readBitSet("preloadKeys"); if (!reader.isLastRead()) return false; @@ -618,7 +610,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 31: - taskNameHash = reader.readInt("taskNameHash"); + skipCompletedVers = reader.readBoolean("skipCompletedVers"); if (!reader.isLastRead()) return false; @@ -626,7 +618,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 32: - topVer = reader.readMessage("topVer"); + subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) return false; @@ -634,7 +626,7 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 33: - mvccSnapshot = reader.readMessage("mvccSnapshot"); + taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) return false; @@ -642,7 +634,15 @@ public boolean skipCompletedVersion() { reader.incrementState(); case 34: - counters = reader.readCollection("counters", MessageCollectionItemType.MSG); + topVer = reader.readAffinityTopologyVersion("topVer"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 35: + updCntrs = reader.readCollection("updCntrs", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -661,7 +661,7 @@ public boolean skipCompletedVersion() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 35; + return 36; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java index 0c2bf8198bac6..fcb14a34c58e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java @@ -245,31 +245,31 @@ public void addPreloadEntry(GridCacheEntryInfo info) { } switch (writer.state()) { - case 10: + case 11: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeMap("invalidParts", invalidParts, MessageCollectionItemType.INT, MessageCollectionItemType.INT_ARR)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeCollection("nearEvicted", nearEvicted, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeCollection("preloadEntries", preloadEntries, MessageCollectionItemType.MSG)) return false; @@ -291,7 +291,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) { return false; switch (reader.state()) { - case 10: + case 11: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -299,7 +299,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) { reader.incrementState(); - case 11: + case 12: invalidParts = reader.readMap("invalidParts", MessageCollectionItemType.INT, MessageCollectionItemType.INT_ARR, false); if (!reader.isLastRead()) @@ -307,7 +307,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) { reader.incrementState(); - case 12: + case 13: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -315,7 +315,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) { reader.incrementState(); - case 13: + case 14: nearEvicted = reader.readCollection("nearEvicted", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -323,7 +323,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) { reader.incrementState(); - case 14: + case 15: preloadEntries = reader.readCollection("preloadEntries", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -343,7 +343,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 15; + return 16; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistRequest.java index b3aa56ddf3cc5..27b7c81aa1479 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistRequest.java @@ -247,43 +247,43 @@ else if (val instanceof GridInvokeValue) } switch (writer.state()) { - case 3: + case 4: if (!writer.writeInt("batchId", batchId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeIgniteUuid("dhtFutId", dhtFutId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeMessage("lockVer", lockVer)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeInt("mvccOpCnt", mvccOpCnt)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeByte("op", op != null ? (byte)op.ordinal() : -1)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG)) return false; @@ -305,7 +305,7 @@ else if (val instanceof GridInvokeValue) return false; switch (reader.state()) { - case 3: + case 4: batchId = reader.readInt("batchId"); if (!reader.isLastRead()) @@ -313,7 +313,7 @@ else if (val instanceof GridInvokeValue) reader.incrementState(); - case 4: + case 5: dhtFutId = reader.readIgniteUuid("dhtFutId"); if (!reader.isLastRead()) @@ -321,7 +321,7 @@ else if (val instanceof GridInvokeValue) reader.incrementState(); - case 5: + case 6: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -329,7 +329,7 @@ else if (val instanceof GridInvokeValue) reader.incrementState(); - case 6: + case 7: lockVer = reader.readMessage("lockVer"); if (!reader.isLastRead()) @@ -337,7 +337,7 @@ else if (val instanceof GridInvokeValue) reader.incrementState(); - case 7: + case 8: mvccOpCnt = reader.readInt("mvccOpCnt"); if (!reader.isLastRead()) @@ -345,7 +345,7 @@ else if (val instanceof GridInvokeValue) reader.incrementState(); - case 8: + case 9: byte opOrd; opOrd = reader.readByte("op"); @@ -357,7 +357,7 @@ else if (val instanceof GridInvokeValue) reader.incrementState(); - case 9: + case 10: vals = reader.readCollection("vals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -372,7 +372,7 @@ else if (val instanceof GridInvokeValue) /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java index f3b4aa7c95824..18f4baa52808d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java @@ -117,7 +117,7 @@ public int batchId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 6; + return 7; } /** {@inheritDoc} */ @@ -135,19 +135,19 @@ public int batchId() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeInt("batchId", batchId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeIgniteUuid("futId", futId)) return false; @@ -169,7 +169,7 @@ public int batchId() { return false; switch (reader.state()) { - case 3: + case 4: batchId = reader.readInt("batchId"); if (!reader.isLastRead()) @@ -177,7 +177,7 @@ public int batchId() { reader.incrementState(); - case 4: + case 5: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -185,7 +185,7 @@ public int batchId() { reader.incrementState(); - case 5: + case 6: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryFirstEnlistRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryFirstEnlistRequest.java index 5c1bf6cb3e61f..2220b29720f8a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryFirstEnlistRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryFirstEnlistRequest.java @@ -208,56 +208,56 @@ public long cleanupVersion() { } switch (writer.state()) { - case 10: + case 11: if (!writer.writeLong("cleanupVer", cleanupVer)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeLong("cntr", cntr)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeLong("crdVer", crdVer)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeUuid("nearNodeId", nearNodeId)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeMessage("nearXidVer", nearXidVer)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 18: - if (!writer.writeMessage("topVer", topVer)) + case 19: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -278,7 +278,7 @@ public long cleanupVersion() { return false; switch (reader.state()) { - case 10: + case 11: cleanupVer = reader.readLong("cleanupVer"); if (!reader.isLastRead()) @@ -286,7 +286,7 @@ public long cleanupVersion() { reader.incrementState(); - case 11: + case 12: cntr = reader.readLong("cntr"); if (!reader.isLastRead()) @@ -294,7 +294,7 @@ public long cleanupVersion() { reader.incrementState(); - case 12: + case 13: crdVer = reader.readLong("crdVer"); if (!reader.isLastRead()) @@ -302,7 +302,7 @@ public long cleanupVersion() { reader.incrementState(); - case 13: + case 14: nearNodeId = reader.readUuid("nearNodeId"); if (!reader.isLastRead()) @@ -310,7 +310,7 @@ public long cleanupVersion() { reader.incrementState(); - case 14: + case 15: nearXidVer = reader.readMessage("nearXidVer"); if (!reader.isLastRead()) @@ -318,7 +318,7 @@ public long cleanupVersion() { reader.incrementState(); - case 15: + case 16: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -326,7 +326,7 @@ public long cleanupVersion() { reader.incrementState(); - case 16: + case 17: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -334,7 +334,7 @@ public long cleanupVersion() { reader.incrementState(); - case 17: + case 18: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -342,8 +342,8 @@ public long cleanupVersion() { reader.incrementState(); - case 18: - topVer = reader.readMessage("topVer"); + case 19: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -357,7 +357,7 @@ public long cleanupVersion() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 19; + return 20; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java index 5671d7fd0f044..3bc4de01f6b17 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java @@ -113,7 +113,7 @@ public void addNearKey(KeyCacheObject key) } switch (writer.state()) { - case 8: + case 9: if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG)) return false; @@ -135,7 +135,7 @@ public void addNearKey(KeyCacheObject key) return false; switch (reader.state()) { - case 8: + case 9: nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -155,6 +155,6 @@ public void addNearKey(KeyCacheObject key) /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 9; + return 10; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCountersMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCountersMessage.java index 3e5953fcb7ca6..f30d51d9016d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCountersMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCountersMessage.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageReader; @@ -28,6 +29,7 @@ /** * Partition update counters message. */ +@IgniteCodeGeneratingFail public class PartitionUpdateCountersMessage implements Message { /** */ private static final int ITEM_SIZE = 4 /* partition */ + 8 /* initial counter */ + 8 /* updates count */; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java index a5e9feb1255a4..0096f012d0fe5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java @@ -484,7 +484,7 @@ final boolean isFlag(int mask) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 12; + return 13; } /** {@inheritDoc} */ @@ -502,55 +502,55 @@ final boolean isFlag(int mask) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeLong("nearFutId", nearFutId)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeUuid("nearNodeId", nearNodeId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 10: - if (!writer.writeMessage("topVer", topVer)) + case 11: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeMessage("writeVer", writeVer)) return false; @@ -572,7 +572,7 @@ final boolean isFlag(int mask) { return false; switch (reader.state()) { - case 3: + case 4: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -580,7 +580,7 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 4: + case 5: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -588,7 +588,7 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 5: + case 6: nearFutId = reader.readLong("nearFutId"); if (!reader.isLastRead()) @@ -596,7 +596,7 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 6: + case 7: nearNodeId = reader.readUuid("nearNodeId"); if (!reader.isLastRead()) @@ -604,7 +604,7 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 7: + case 8: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -612,7 +612,7 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 8: + case 9: byte syncModeOrd; syncModeOrd = reader.readByte("syncMode"); @@ -624,7 +624,7 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 9: + case 10: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -632,15 +632,15 @@ final boolean isFlag(int mask) { reader.incrementState(); - case 10: - topVer = reader.readMessage("topVer"); + case 11: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 11: + case 12: writeVer = reader.readMessage("writeVer"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java index 0c069da80082e..ee5eac15a6c45 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java @@ -119,7 +119,7 @@ GridLongList futureIds() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeMessage("futIds", futIds)) return false; @@ -141,7 +141,7 @@ GridLongList futureIds() { return false; switch (reader.state()) { - case 3: + case 4: futIds = reader.readMessage("futIds"); if (!reader.isLastRead()) @@ -161,7 +161,7 @@ GridLongList futureIds() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 4; + return 5; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java index 71d23216f0e2b..8f11ead53adff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java @@ -171,7 +171,7 @@ public long futureId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 8; + return 9; } /** {@inheritDoc} */ @@ -210,31 +210,31 @@ public long futureId() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeMessage("errs", errs)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeInt("partId", partId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeUuid("primaryId", primaryId)) return false; @@ -256,7 +256,7 @@ public long futureId() { return false; switch (reader.state()) { - case 3: + case 4: errs = reader.readMessage("errs"); if (!reader.isLastRead()) @@ -264,7 +264,7 @@ public long futureId() { reader.incrementState(); - case 4: + case 5: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -272,7 +272,7 @@ public long futureId() { reader.incrementState(); - case 5: + case 6: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -280,7 +280,7 @@ public long futureId() { reader.incrementState(); - case 6: + case 7: partId = reader.readInt("partId"); if (!reader.isLastRead()) @@ -288,7 +288,7 @@ public long futureId() { reader.incrementState(); - case 7: + case 8: primaryId = reader.readUuid("primaryId"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java index 19b24b052d795..16be80e2e5a71 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java @@ -373,25 +373,25 @@ private void near(boolean near) { } switch (writer.state()) { - case 12: + case 13: if (!writer.writeMessage("key", key)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeMessage("prevVal", prevVal)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeLong("updateCntr", updateCntr)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeMessage("val", val)) return false; @@ -413,7 +413,7 @@ private void near(boolean near) { return false; switch (reader.state()) { - case 12: + case 13: key = reader.readMessage("key"); if (!reader.isLastRead()) @@ -421,7 +421,7 @@ private void near(boolean near) { reader.incrementState(); - case 13: + case 14: prevVal = reader.readMessage("prevVal"); if (!reader.isLastRead()) @@ -429,7 +429,7 @@ private void near(boolean near) { reader.incrementState(); - case 14: + case 15: updateCntr = reader.readLong("updateCntr"); if (!reader.isLastRead()) @@ -437,7 +437,7 @@ private void near(boolean near) { reader.incrementState(); - case 15: + case 16: val = reader.readMessage("val"); if (!reader.isLastRead()) @@ -487,7 +487,7 @@ private void finishUnmarshalObject(@Nullable CacheObject obj, GridCacheContext c /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 16; + return 17; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java index 30be9dcc30c22..67281f756267a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java @@ -558,97 +558,97 @@ else if (conflictVers != null) } switch (writer.state()) { - case 12: + case 13: if (!writer.writeMessage("conflictExpireTimes", conflictExpireTimes)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeCollection("conflictVers", conflictVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeCollection("entryProcessorsBytes", entryProcessorsBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeBoolean("forceTransformBackups", forceTransformBackups)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeObjectArray("invokeArgsBytes", invokeArgsBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeCollection("nearEntryProcessorsBytes", nearEntryProcessorsBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 19: + case 20: if (!writer.writeMessage("nearExpireTimes", nearExpireTimes)) return false; writer.incrementState(); - case 20: + case 21: if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 21: + case 22: if (!writer.writeMessage("nearTtls", nearTtls)) return false; writer.incrementState(); - case 22: + case 23: if (!writer.writeCollection("nearVals", nearVals, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 23: + case 24: if (!writer.writeMessage("obsoleteIndexes", obsoleteIndexes)) return false; writer.incrementState(); - case 24: + case 25: if (!writer.writeCollection("prevVals", prevVals, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 25: + case 26: if (!writer.writeMessage("ttls", ttls)) return false; writer.incrementState(); - case 26: + case 27: if (!writer.writeMessage("updateCntrs", updateCntrs)) return false; writer.incrementState(); - case 27: + case 28: if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG)) return false; @@ -670,7 +670,7 @@ else if (conflictVers != null) return false; switch (reader.state()) { - case 12: + case 13: conflictExpireTimes = reader.readMessage("conflictExpireTimes"); if (!reader.isLastRead()) @@ -678,7 +678,7 @@ else if (conflictVers != null) reader.incrementState(); - case 13: + case 14: conflictVers = reader.readCollection("conflictVers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -686,7 +686,7 @@ else if (conflictVers != null) reader.incrementState(); - case 14: + case 15: entryProcessorsBytes = reader.readCollection("entryProcessorsBytes", MessageCollectionItemType.BYTE_ARR); if (!reader.isLastRead()) @@ -694,7 +694,7 @@ else if (conflictVers != null) reader.incrementState(); - case 15: + case 16: forceTransformBackups = reader.readBoolean("forceTransformBackups"); if (!reader.isLastRead()) @@ -702,7 +702,7 @@ else if (conflictVers != null) reader.incrementState(); - case 16: + case 17: invokeArgsBytes = reader.readObjectArray("invokeArgsBytes", MessageCollectionItemType.BYTE_ARR, byte[].class); if (!reader.isLastRead()) @@ -710,7 +710,7 @@ else if (conflictVers != null) reader.incrementState(); - case 17: + case 18: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -718,7 +718,7 @@ else if (conflictVers != null) reader.incrementState(); - case 18: + case 19: nearEntryProcessorsBytes = reader.readCollection("nearEntryProcessorsBytes", MessageCollectionItemType.BYTE_ARR); if (!reader.isLastRead()) @@ -726,7 +726,7 @@ else if (conflictVers != null) reader.incrementState(); - case 19: + case 20: nearExpireTimes = reader.readMessage("nearExpireTimes"); if (!reader.isLastRead()) @@ -734,7 +734,7 @@ else if (conflictVers != null) reader.incrementState(); - case 20: + case 21: nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -742,7 +742,7 @@ else if (conflictVers != null) reader.incrementState(); - case 21: + case 22: nearTtls = reader.readMessage("nearTtls"); if (!reader.isLastRead()) @@ -750,7 +750,7 @@ else if (conflictVers != null) reader.incrementState(); - case 22: + case 23: nearVals = reader.readCollection("nearVals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -758,7 +758,7 @@ else if (conflictVers != null) reader.incrementState(); - case 23: + case 24: obsoleteIndexes = reader.readMessage("obsoleteIndexes"); if (!reader.isLastRead()) @@ -766,7 +766,7 @@ else if (conflictVers != null) reader.incrementState(); - case 24: + case 25: prevVals = reader.readCollection("prevVals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -774,7 +774,7 @@ else if (conflictVers != null) reader.incrementState(); - case 25: + case 26: ttls = reader.readMessage("ttls"); if (!reader.isLastRead()) @@ -782,7 +782,7 @@ else if (conflictVers != null) reader.incrementState(); - case 26: + case 27: updateCntrs = reader.readMessage("updateCntrs"); if (!reader.isLastRead()) @@ -790,7 +790,7 @@ else if (conflictVers != null) reader.incrementState(); - case 27: + case 28: vals = reader.readCollection("vals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -816,7 +816,7 @@ else if (conflictVers != null) /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 28; + return 29; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java index 70bf6f5648b41..21efbb1350b3d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java @@ -179,25 +179,25 @@ public void nearEvicted(List nearEvicted) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeMessage("errs", errs)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeCollection("nearEvicted", nearEvicted, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeInt("partId", partId)) return false; @@ -219,7 +219,7 @@ public void nearEvicted(List nearEvicted) { return false; switch (reader.state()) { - case 3: + case 4: errs = reader.readMessage("errs"); if (!reader.isLastRead()) @@ -227,7 +227,7 @@ public void nearEvicted(List nearEvicted) { reader.incrementState(); - case 4: + case 5: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -235,7 +235,7 @@ public void nearEvicted(List nearEvicted) { reader.incrementState(); - case 5: + case 6: nearEvicted = reader.readCollection("nearEvicted", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -243,7 +243,7 @@ public void nearEvicted(List nearEvicted) { reader.incrementState(); - case 6: + case 7: partId = reader.readInt("partId"); if (!reader.isLastRead()) @@ -263,7 +263,7 @@ public void nearEvicted(List nearEvicted) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 7; + return 8; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java index 62618f81e6af5..64fe1eef8fe6c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java @@ -528,7 +528,7 @@ abstract void addUpdateEntry(KeyCacheObject key, /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ @@ -546,44 +546,44 @@ abstract void addUpdateEntry(KeyCacheObject key, } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeByte("op", op != null ? (byte)op.ordinal() : -1)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 9: - if (!writer.writeMessage("topVer", topVer)) + case 10: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -604,7 +604,7 @@ abstract void addUpdateEntry(KeyCacheObject key, return false; switch (reader.state()) { - case 3: + case 4: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -612,7 +612,7 @@ abstract void addUpdateEntry(KeyCacheObject key, reader.incrementState(); - case 4: + case 5: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -620,7 +620,7 @@ abstract void addUpdateEntry(KeyCacheObject key, reader.incrementState(); - case 5: + case 6: byte opOrd; opOrd = reader.readByte("op"); @@ -632,7 +632,7 @@ abstract void addUpdateEntry(KeyCacheObject key, reader.incrementState(); - case 6: + case 7: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -640,7 +640,7 @@ abstract void addUpdateEntry(KeyCacheObject key, reader.incrementState(); - case 7: + case 8: byte syncModeOrd; syncModeOrd = reader.readByte("syncMode"); @@ -652,7 +652,7 @@ abstract void addUpdateEntry(KeyCacheObject key, reader.incrementState(); - case 8: + case 9: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -660,8 +660,8 @@ abstract void addUpdateEntry(KeyCacheObject key, reader.incrementState(); - case 9: - topVer = reader.readMessage("topVer"); + case 10: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java index 96be0233c308a..a19e28029b89f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java @@ -101,7 +101,7 @@ GridNearAtomicAbstractUpdateRequest updateRequest() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 5; + return 6; } /** {@inheritDoc} */ @@ -119,13 +119,13 @@ GridNearAtomicAbstractUpdateRequest updateRequest() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeInt("partId", partId)) return false; @@ -147,7 +147,7 @@ GridNearAtomicAbstractUpdateRequest updateRequest() { return false; switch (reader.state()) { - case 3: + case 4: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -155,7 +155,7 @@ GridNearAtomicAbstractUpdateRequest updateRequest() { reader.incrementState(); - case 4: + case 5: partId = reader.readInt("partId"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java index d6956a64dae69..170586b22d589 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java @@ -435,55 +435,55 @@ else if (conflictVers != null) } switch (writer.state()) { - case 10: + case 11: if (!writer.writeMessage("conflictExpireTimes", conflictExpireTimes)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeMessage("conflictTtls", conflictTtls)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeCollection("conflictVers", conflictVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeCollection("entryProcessorsBytes", entryProcessorsBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeByteArray("expiryPlcBytes", expiryPlcBytes)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeObjectArray("invokeArgsBytes", invokeArgsBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG)) return false; @@ -505,7 +505,7 @@ else if (conflictVers != null) return false; switch (reader.state()) { - case 10: + case 11: conflictExpireTimes = reader.readMessage("conflictExpireTimes"); if (!reader.isLastRead()) @@ -513,7 +513,7 @@ else if (conflictVers != null) reader.incrementState(); - case 11: + case 12: conflictTtls = reader.readMessage("conflictTtls"); if (!reader.isLastRead()) @@ -521,7 +521,7 @@ else if (conflictVers != null) reader.incrementState(); - case 12: + case 13: conflictVers = reader.readCollection("conflictVers", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -529,7 +529,7 @@ else if (conflictVers != null) reader.incrementState(); - case 13: + case 14: entryProcessorsBytes = reader.readCollection("entryProcessorsBytes", MessageCollectionItemType.BYTE_ARR); if (!reader.isLastRead()) @@ -537,7 +537,7 @@ else if (conflictVers != null) reader.incrementState(); - case 14: + case 15: expiryPlcBytes = reader.readByteArray("expiryPlcBytes"); if (!reader.isLastRead()) @@ -545,7 +545,7 @@ else if (conflictVers != null) reader.incrementState(); - case 15: + case 16: filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class); if (!reader.isLastRead()) @@ -553,7 +553,7 @@ else if (conflictVers != null) reader.incrementState(); - case 16: + case 17: invokeArgsBytes = reader.readObjectArray("invokeArgsBytes", MessageCollectionItemType.BYTE_ARR, byte[].class); if (!reader.isLastRead()) @@ -561,7 +561,7 @@ else if (conflictVers != null) reader.incrementState(); - case 17: + case 18: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -569,7 +569,7 @@ else if (conflictVers != null) reader.incrementState(); - case 18: + case 19: vals = reader.readCollection("vals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -601,7 +601,7 @@ else if (conflictVers != null) /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 19; + return 20; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java index 5c66bc46894b2..c7076988a659f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java @@ -155,7 +155,7 @@ public GridNearAtomicSingleUpdateFilterRequest() { } switch (writer.state()) { - case 12: + case 13: if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG)) return false; @@ -177,7 +177,7 @@ public GridNearAtomicSingleUpdateFilterRequest() { return false; switch (reader.state()) { - case 12: + case 13: filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class); if (!reader.isLastRead()) @@ -197,7 +197,7 @@ public GridNearAtomicSingleUpdateFilterRequest() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 13; + return 14; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java index 865d6f8664e9e..ee3d2a4fe036a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java @@ -225,13 +225,13 @@ public GridNearAtomicSingleUpdateInvokeRequest() { } switch (writer.state()) { - case 12: + case 13: if (!writer.writeByteArray("entryProcessorBytes", entryProcessorBytes)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeObjectArray("invokeArgsBytes", invokeArgsBytes, MessageCollectionItemType.BYTE_ARR)) return false; @@ -253,7 +253,7 @@ public GridNearAtomicSingleUpdateInvokeRequest() { return false; switch (reader.state()) { - case 12: + case 13: entryProcessorBytes = reader.readByteArray("entryProcessorBytes"); if (!reader.isLastRead()) @@ -261,7 +261,7 @@ public GridNearAtomicSingleUpdateInvokeRequest() { reader.incrementState(); - case 13: + case 14: invokeArgsBytes = reader.readObjectArray("invokeArgsBytes", MessageCollectionItemType.BYTE_ARR, byte[].class); if (!reader.isLastRead()) @@ -276,7 +276,7 @@ public GridNearAtomicSingleUpdateInvokeRequest() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 14; + return 15; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java index dd3a7bedb7de9..83ec4565f49ce 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java @@ -247,13 +247,13 @@ public GridNearAtomicSingleUpdateRequest() { } switch (writer.state()) { - case 10: + case 11: if (!writer.writeMessage("key", key)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeMessage("val", val)) return false; @@ -275,7 +275,7 @@ public GridNearAtomicSingleUpdateRequest() { return false; switch (reader.state()) { - case 10: + case 11: key = reader.readMessage("key"); if (!reader.isLastRead()) @@ -283,7 +283,7 @@ public GridNearAtomicSingleUpdateRequest() { reader.incrementState(); - case 11: + case 12: val = reader.readMessage("val"); if (!reader.isLastRead()) @@ -311,7 +311,7 @@ public GridNearAtomicSingleUpdateRequest() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 12; + return 13; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java index 37fe824a39f99..6dccd8b9cdc9b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java @@ -405,43 +405,43 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeMessage("errs", errs)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeCollection("mapping", mapping, MessageCollectionItemType.UUID)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeMessage("nearUpdates", nearUpdates)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeInt("partId", partId)) return false; writer.incrementState(); - case 8: - if (!writer.writeMessage("remapTopVer", remapTopVer)) + case 9: + if (!writer.writeAffinityTopologyVersion("remapTopVer", remapTopVer)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeMessage("ret", ret)) return false; @@ -463,7 +463,7 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { return false; switch (reader.state()) { - case 3: + case 4: errs = reader.readMessage("errs"); if (!reader.isLastRead()) @@ -471,7 +471,7 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { reader.incrementState(); - case 4: + case 5: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -479,7 +479,7 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { reader.incrementState(); - case 5: + case 6: mapping = reader.readCollection("mapping", MessageCollectionItemType.UUID); if (!reader.isLastRead()) @@ -487,7 +487,7 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { reader.incrementState(); - case 6: + case 7: nearUpdates = reader.readMessage("nearUpdates"); if (!reader.isLastRead()) @@ -495,7 +495,7 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { reader.incrementState(); - case 7: + case 8: partId = reader.readInt("partId"); if (!reader.isLastRead()) @@ -503,15 +503,15 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { reader.incrementState(); - case 8: - remapTopVer = reader.readMessage("remapTopVer"); + case 9: + remapTopVer = reader.readAffinityTopologyVersion("remapTopVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 9: + case 10: ret = reader.readMessage("ret"); if (!reader.isLastRead()) @@ -531,7 +531,7 @@ synchronized void addFailedKeys(Collection keys, Throwable e) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java index 124ae44ca5b30..80c45efc76805 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java @@ -167,26 +167,26 @@ private int keyCount() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 6: - if (!writer.writeMessage("topVer", topVer)) + case 7: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -207,7 +207,7 @@ private int keyCount() { return false; switch (reader.state()) { - case 3: + case 4: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -215,7 +215,7 @@ private int keyCount() { reader.incrementState(); - case 4: + case 5: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -223,7 +223,7 @@ private int keyCount() { reader.incrementState(); - case 5: + case 6: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -231,8 +231,8 @@ private int keyCount() { reader.incrementState(); - case 6: - topVer = reader.readMessage("topVer"); + case 7: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -251,7 +251,7 @@ private int keyCount() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 7; + return 8; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java index 977e9ba41eef2..ab85df3e94622 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java @@ -213,31 +213,31 @@ public void addInfo(GridCacheEntryInfo info) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeCollection("infos", infos, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeCollection("missedKeys", missedKeys, MessageCollectionItemType.MSG)) return false; @@ -259,7 +259,7 @@ public void addInfo(GridCacheEntryInfo info) { return false; switch (reader.state()) { - case 3: + case 4: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -267,7 +267,7 @@ public void addInfo(GridCacheEntryInfo info) { reader.incrementState(); - case 4: + case 5: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -275,7 +275,7 @@ public void addInfo(GridCacheEntryInfo info) { reader.incrementState(); - case 5: + case 6: infos = reader.readCollection("infos", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -283,7 +283,7 @@ public void addInfo(GridCacheEntryInfo info) { reader.incrementState(); - case 6: + case 7: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -291,7 +291,7 @@ public void addInfo(GridCacheEntryInfo info) { reader.incrementState(); - case 7: + case 8: missedKeys = reader.readCollection("missedKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -311,7 +311,7 @@ public void addInfo(GridCacheEntryInfo info) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 8; + return 9; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java index 46e9ceb004830..cd7741b554fa7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java @@ -285,49 +285,49 @@ Long partitionCounter(int part) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeCollection("historicalParts", historicalParts, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeCollection("parts", parts, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeMap("partsCntrs", partsCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 7: - if (!writer.writeMessage("topVer", topVer)) + case 8: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeByteArray("topicBytes", topicBytes)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeLong("updateSeq", updateSeq)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeInt("workerId", workerId)) return false; @@ -349,7 +349,7 @@ Long partitionCounter(int part) { return false; switch (reader.state()) { - case 3: + case 4: historicalParts = reader.readCollection("historicalParts", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -357,7 +357,7 @@ Long partitionCounter(int part) { reader.incrementState(); - case 4: + case 5: parts = reader.readCollection("parts", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -365,7 +365,7 @@ Long partitionCounter(int part) { reader.incrementState(); - case 5: + case 6: partsCntrs = reader.readMap("partsCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false); if (!reader.isLastRead()) @@ -373,7 +373,7 @@ Long partitionCounter(int part) { reader.incrementState(); - case 6: + case 7: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -381,15 +381,15 @@ Long partitionCounter(int part) { reader.incrementState(); - case 7: - topVer = reader.readMessage("topVer"); + case 8: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 8: + case 9: topicBytes = reader.readByteArray("topicBytes"); if (!reader.isLastRead()) @@ -397,7 +397,7 @@ Long partitionCounter(int part) { reader.incrementState(); - case 9: + case 10: updateSeq = reader.readLong("updateSeq"); if (!reader.isLastRead()) @@ -405,7 +405,7 @@ Long partitionCounter(int part) { reader.incrementState(); - case 10: + case 11: workerId = reader.readInt("workerId"); if (!reader.isLastRead()) @@ -415,7 +415,7 @@ Long partitionCounter(int part) { } - return reader.afterMessageRead(GridDhtPartitionDemandMessage.class); + return reader.afterMessageRead(GridDhtPartitionDemandLegacyMessage.class); } /** {@inheritDoc} */ @@ -425,7 +425,7 @@ Long partitionCounter(int part) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 11; + return 12; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java index dc6162bc321f2..bae326424d0fb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; import org.apache.ignite.internal.processors.cache.GridCacheMessage; @@ -35,6 +36,7 @@ /** * Partition demand request. */ +@IgniteCodeGeneratingFail public class GridDhtPartitionDemandMessage extends GridCacheGroupIdMessage { /** */ private static final long serialVersionUID = 0L; @@ -259,37 +261,37 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByteArray("partsBytes", partsBytes)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 5: - if (!writer.writeMessage("topVer", topVer)) + case 6: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeByteArray("topicBytes", topicBytes)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeLong("rebalanceId", rebalanceId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeInt("workerId", workerId)) return false; @@ -311,7 +313,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { return false; switch (reader.state()) { - case 3: + case 4: partsBytes = reader.readByteArray("partsBytes"); if (!reader.isLastRead()) @@ -319,7 +321,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { reader.incrementState(); - case 4: + case 5: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -327,15 +329,15 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { reader.incrementState(); - case 5: - topVer = reader.readMessage("topVer"); + case 6: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 6: + case 7: topicBytes = reader.readByteArray("topicBytes"); if (!reader.isLastRead()) @@ -343,7 +345,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { reader.incrementState(); - case 7: + case 8: rebalanceId = reader.readLong("rebalanceId"); if (!reader.isLastRead()) @@ -351,7 +353,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { reader.incrementState(); - case 8: + case 9: workerId = reader.readInt("workerId"); if (!reader.isLastRead()) @@ -371,7 +373,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 9; + return 10; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java index 741386b40707b..2206f2e372fad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java @@ -248,7 +248,7 @@ public boolean isLeft() { writer.incrementState(); case 2: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -283,7 +283,7 @@ public boolean isLeft() { reader.incrementState(); case 2: - topVer = reader.readMessage("topVer"); + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -311,4 +311,4 @@ public boolean isLeft() { "nodeId", U.id8(nodeId), "evt", U.gridEventName(evt)); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java index 284700ad3a44f..3034fb9daaf21 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java @@ -28,6 +28,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.GridDirectCollection; import org.apache.ignite.internal.GridDirectMap; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryInfoCollection; import org.apache.ignite.internal.processors.cache.CacheGroupContext; @@ -45,6 +46,7 @@ /** * Partition supply message. */ +@IgniteCodeGeneratingFail public class GridDhtPartitionSupplyMessage extends GridCacheGroupIdMessage implements GridCacheDeployable { /** */ private static final long serialVersionUID = 0L; @@ -282,55 +284,55 @@ public int size() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeCollection("clean", clean, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("estimatedKeysCnt", estimatedKeysCnt)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeMap("infos", infos, MessageCollectionItemType.INT, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeMap("keysPerCache", keysPerCache, MessageCollectionItemType.INT, MessageCollectionItemType.LONG)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeMap("last", last, MessageCollectionItemType.INT, MessageCollectionItemType.LONG)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeCollection("missed", missed, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeInt("msgSize", msgSize)) return false; writer.incrementState(); - case 10: - if (!writer.writeMessage("topVer", topVer)) + case 11: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 11: + case 12: // Keep 'updateSeq' name for compatibility. if (!writer.writeLong("updateSeq", rebalanceId)) return false; @@ -353,7 +355,7 @@ public int size() { return false; switch (reader.state()) { - case 3: + case 4: clean = reader.readCollection("clean", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -361,7 +363,7 @@ public int size() { reader.incrementState(); - case 4: + case 5: estimatedKeysCnt = reader.readLong("estimatedKeysCnt"); if (!reader.isLastRead()) @@ -369,7 +371,7 @@ public int size() { reader.incrementState(); - case 5: + case 6: infos = reader.readMap("infos", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false); if (!reader.isLastRead()) @@ -377,7 +379,7 @@ public int size() { reader.incrementState(); - case 6: + case 7: keysPerCache = reader.readMap("keysPerCache", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false); if (!reader.isLastRead()) @@ -385,7 +387,7 @@ public int size() { reader.incrementState(); - case 7: + case 8: last = reader.readMap("last", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false); if (!reader.isLastRead()) @@ -393,7 +395,7 @@ public int size() { reader.incrementState(); - case 8: + case 9: missed = reader.readCollection("missed", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -401,7 +403,7 @@ public int size() { reader.incrementState(); - case 9: + case 10: msgSize = reader.readInt("msgSize"); if (!reader.isLastRead()) @@ -409,15 +411,15 @@ public int size() { reader.incrementState(); - case 10: - topVer = reader.readMessage("topVer"); + case 11: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 11: + case 12: // Keep 'updateSeq' name for compatibility. rebalanceId = reader.readLong("updateSeq"); @@ -438,7 +440,7 @@ public int size() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 12; + return 13; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java index b6bff0e823506..b209cdba342ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java @@ -101,7 +101,7 @@ public GridDhtPartitionSupplyMessageV2( } switch (writer.state()) { - case 12: + case 13: if (!writer.writeByteArray("errBytes", errBytes)) return false; @@ -123,7 +123,7 @@ public GridDhtPartitionSupplyMessageV2( return false; switch (reader.state()) { - case 12: + case 13: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -148,6 +148,6 @@ public GridDhtPartitionSupplyMessageV2( /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 13; + return 14; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java index 84cc792fe22ea..e2884e1751508 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java @@ -145,7 +145,7 @@ public boolean restoreState() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 5; + return 6; } /** {@inheritDoc} */ @@ -163,19 +163,19 @@ public boolean restoreState() { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeMessage("exchId", exchId)) return false; writer.incrementState(); - case 3: + case 4: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeMessage("lastVer", lastVer)) return false; @@ -197,7 +197,7 @@ public boolean restoreState() { return false; switch (reader.state()) { - case 2: + case 3: exchId = reader.readMessage("exchId"); if (!reader.isLastRead()) @@ -205,7 +205,7 @@ public boolean restoreState() { reader.incrementState(); - case 3: + case 4: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -213,7 +213,7 @@ public boolean restoreState() { reader.incrementState(); - case 4: + case 5: lastVer = reader.readMessage("lastVer"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 08fbbf90afd10..ec90d61f89d90 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -567,6 +567,16 @@ public boolean changedBaseline() { return exchActions != null && exchActions.changedBaseline(); } + /** {@inheritDoc} */ + @Override public boolean changedAffinity() { + DiscoveryEvent firstDiscoEvt0 = firstDiscoEvt; + + assert firstDiscoEvt0 != null; + + return firstDiscoEvt0.type() == DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT + || !firstDiscoEvt0.eventNode().isClient() || firstDiscoEvt0.eventNode().isLocal(); + } + /** * @return {@code True} if there are caches to start. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index a63ab70e28d53..7fccfac842a4c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -595,74 +595,74 @@ public void topologyVersion(AffinityTopologyVersion topVer) { } switch (writer.state()) { - case 5: + case 6: if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeByteArray("errsBytes", errsBytes)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeMap("idealAffDiff", idealAffDiff, MessageCollectionItemType.INT, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeMap("joinedNodeAff", joinedNodeAff, MessageCollectionItemType.INT, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeByteArray("partCntrsBytes2", partCntrsBytes2)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeByteArray("partsBytes", partsBytes)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes)) return false; writer.incrementState(); - case 15: - if (!writer.writeMessage("resTopVer", resTopVer)) + case 16: + if (!writer.writeAffinityTopologyVersion("resTopVer", resTopVer)) return false; writer.incrementState(); - case 16: - if (!writer.writeMessage("topVer", topVer)) + case 17: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -683,7 +683,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { return false; switch (reader.state()) { - case 5: + case 6: dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false); if (!reader.isLastRead()) @@ -691,7 +691,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 6: + case 7: errsBytes = reader.readByteArray("errsBytes"); if (!reader.isLastRead()) @@ -699,7 +699,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 7: + case 8: idealAffDiff = reader.readMap("idealAffDiff", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false); if (!reader.isLastRead()) @@ -707,7 +707,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 8: + case 9: joinedNodeAff = reader.readMap("joinedNodeAff", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false); if (!reader.isLastRead()) @@ -715,7 +715,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 9: + case 10: partCntrsBytes = reader.readByteArray("partCntrsBytes"); if (!reader.isLastRead()) @@ -723,7 +723,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 10: + case 11: partCntrsBytes2 = reader.readByteArray("partCntrsBytes2"); if (!reader.isLastRead()) @@ -731,7 +731,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 11: + case 12: partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes"); if (!reader.isLastRead()) @@ -739,7 +739,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 12: + case 13: partsBytes = reader.readByteArray("partsBytes"); if (!reader.isLastRead()) @@ -747,7 +747,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 13: + case 14: partsSizesBytes = reader.readByteArray("partsSizesBytes"); if (!reader.isLastRead()) @@ -755,7 +755,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 14: + case 15: partsToReloadBytes = reader.readByteArray("partsToReloadBytes"); if (!reader.isLastRead()) @@ -763,16 +763,16 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 15: - resTopVer = reader.readMessage("resTopVer"); + case 16: + resTopVer = reader.readAffinityTopologyVersion("resTopVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 16: - topVer = reader.readMessage("topVer"); + case 17: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -791,7 +791,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 17; + return 18; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index 088fb31d7f715..b5de155255cc4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -473,65 +473,66 @@ public void setError(Exception ex) { } switch (writer.state()) { - case 5: + case 6: + if (!writer.writeMessage("activeQryTrackers", activeQryTrackers)) + return false; + + writer.incrementState(); + + case 7: if (!writer.writeBoolean("client", client)) return false; writer.incrementState(); - case 6: + case 8: if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 7: + case 9: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 8: + case 10: if (!writer.writeMessage("finishMsg", finishMsg)) return false; writer.incrementState(); - case 9: + case 11: if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 10: + case 12: if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes)) return false; writer.incrementState(); - case 11: + case 13: if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes)) return false; writer.incrementState(); - case 12: + case 14: if (!writer.writeByteArray("partsBytes", partsBytes)) return false; writer.incrementState(); - case 13: + case 15: if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes)) return false; writer.incrementState(); - case 14: - if (!writer.writeMessage("activeQryTrackers", activeQryTrackers)) - return false; - - writer.incrementState(); } return true; @@ -548,7 +549,15 @@ public void setError(Exception ex) { return false; switch (reader.state()) { - case 5: + case 6: + activeQryTrackers = reader.readMessage("activeQryTrackers"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 7: client = reader.readBoolean("client"); if (!reader.isLastRead()) @@ -556,7 +565,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 6: + case 8: dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false); if (!reader.isLastRead()) @@ -564,7 +573,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 7: + case 9: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -572,7 +581,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 8: + case 10: finishMsg = reader.readMessage("finishMsg"); if (!reader.isLastRead()) @@ -580,7 +589,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 9: + case 11: grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -588,7 +597,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 10: + case 12: partCntrsBytes = reader.readByteArray("partCntrsBytes"); if (!reader.isLastRead()) @@ -596,7 +605,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 11: + case 13: partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes"); if (!reader.isLastRead()) @@ -604,7 +613,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 12: + case 14: partsBytes = reader.readByteArray("partsBytes"); if (!reader.isLastRead()) @@ -612,7 +621,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 13: + case 15: partsSizesBytes = reader.readByteArray("partsSizesBytes"); if (!reader.isLastRead()) @@ -620,13 +629,6 @@ public void setError(Exception ex) { reader.incrementState(); - case 14: - activeQryTrackers = reader.readMessage("activeQryTrackers"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); } return reader.afterMessageRead(GridDhtPartitionsSingleMessage.class); @@ -639,7 +641,7 @@ public void setError(Exception ex) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 15; + return 16; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java index 0be0f37aa22ac..26d3cdeffd684 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java @@ -89,7 +89,7 @@ GridDhtPartitionExchangeId restoreExchangeId() { } switch (writer.state()) { - case 5: + case 6: if (!writer.writeMessage("restoreExchId", restoreExchId)) return false; @@ -111,7 +111,7 @@ GridDhtPartitionExchangeId restoreExchangeId() { return false; switch (reader.state()) { - case 5: + case 6: restoreExchId = reader.readMessage("restoreExchId"); if (!reader.isLastRead()) @@ -131,7 +131,7 @@ GridDhtPartitionExchangeId restoreExchangeId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 6; + return 7; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java index bad1b6137bac5..9c69fdf581856 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java @@ -103,10 +103,11 @@ public boolean isFinal() { writer.incrementState(); case 2: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); + } return true; @@ -137,12 +138,13 @@ public boolean isFinal() { reader.incrementState(); case 2: - topVer = reader.readMessage("topVer"); + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); + } return reader.afterMessageRead(LatchAckMessage.class); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 683a140c5d978..de0b55a0cb183 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -313,9 +313,18 @@ private String mapString(GridDhtPartitionMap map) { /** {@inheritDoc} */ @Override public GridDhtTopologyFuture topologyVersionFuture() { - assert topReadyFut != null; + GridDhtTopologyFuture topReadyFut0 = topReadyFut; - return topReadyFut; + assert topReadyFut0 != null; + + if (!topReadyFut0.changedAffinity()) { + GridDhtTopologyFuture lastFut = ctx.exchange().lastFinishedFuture(); + + if (lastFut != null) + return lastFut; + } + + return topReadyFut0; } /** {@inheritDoc} */ @@ -748,45 +757,47 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) { long updateSeq = this.updateSeq.incrementAndGet(); - for (int p = 0; p < partitions; p++) { - GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true); - - if (partitionLocalNode(p, topVer)) { - // Prepare partition to rebalance if it's not happened on full map update phase. - if (locPart == null || locPart.state() == RENTING || locPart.state() == EVICTED) - locPart = rebalancePartition(p, false); + if (!ctx.localNode().isClient()) { + for (int p = 0; p < partitions; p++) { + GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true); - GridDhtPartitionState state = locPart.state(); + if (partitionLocalNode(p, topVer)) { + // Prepare partition to rebalance if it's not happened on full map update phase. + if (locPart == null || locPart.state() == RENTING || locPart.state() == EVICTED) + locPart = rebalancePartition(p, false); - if (state == MOVING) { - if (grp.rebalanceEnabled()) { - Collection owners = owners(p); + GridDhtPartitionState state = locPart.state(); - // If an owner node left during exchange, then new exchange should be started with detecting lost partitions. - if (!F.isEmpty(owners)) { - if (log.isDebugEnabled()) - log.debug("Will not own partition (there are owners to rebalance from) " + - "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", owners = " + owners + ']'); + if (state == MOVING) { + if (grp.rebalanceEnabled()) { + Collection owners = owners(p); + + // If an owner node left during exchange, then new exchange should be started with detecting lost partitions. + if (!F.isEmpty(owners)) { + if (log.isDebugEnabled()) + log.debug("Will not own partition (there are owners to rebalance from) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", owners = " + owners + ']'); + } } + else + updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); } - else - updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); } - } - else { - if (locPart != null) { - GridDhtPartitionState state = locPart.state(); + else { + if (locPart != null) { + GridDhtPartitionState state = locPart.state(); - if (state == MOVING) { - locPart.rent(false); + if (state == MOVING) { + locPart.rent(false); - updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); + updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer); - changed = true; + changed = true; - if (log.isDebugEnabled()) { - log.debug("Evicting " + state + " partition (it does not belong to affinity) [" + - "grp=" + grp.cacheOrGroupName() + ", p=" + locPart.id() + ']'); + if (log.isDebugEnabled()) { + log.debug("Evicting " + state + " partition (it does not belong to affinity) [" + + "grp=" + grp.cacheOrGroupName() + ", p=" + locPart.id() + ']'); + } } } } @@ -1131,25 +1142,38 @@ else if (loc != null && state == RENTING && !showRenting) { List nodes = null; - if (!topVer.equals(diffFromAffinityVer)) { - LT.warn(log, "Requested topology version does not match calculated diff, will require full iteration to" + - "calculate mapping [grp=" + grp.cacheOrGroupName() + ", topVer=" + topVer + - ", diffVer=" + diffFromAffinityVer + "]"); + AffinityTopologyVersion diffVer = diffFromAffinityVer; - nodes = new ArrayList<>(); + if (!diffVer.equals(topVer)) { + LT.warn(log, "Requested topology version does not match calculated diff, need to check if " + + "affinity has changed [grp=" + grp.cacheOrGroupName() + ", topVer=" + topVer + + ", diffVer=" + diffVer + "]"); - nodes.addAll(affNodes); + boolean affChanged; - for (Map.Entry entry : node2part.entrySet()) { - GridDhtPartitionState state = entry.getValue().get(p); + if (diffVer.compareTo(topVer) < 0) + affChanged = ctx.exchange().affinityChanged(diffVer, topVer); + else + affChanged = ctx.exchange().affinityChanged(topVer, diffVer); - ClusterNode n = ctx.discovery().node(entry.getKey()); + if (affChanged) { + LT.warn(log, "Requested topology version does not match calculated diff, will require full iteration to" + + "calculate mapping [grp=" + grp.cacheOrGroupName() + ", topVer=" + topVer + + ", diffVer=" + diffVer + "]"); - if (n != null && state != null && (state == MOVING || state == OWNING || state == RENTING) - && !nodes.contains(n) && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) { - nodes.add(n); - } + nodes = new ArrayList<>(); + + nodes.addAll(affNodes); + + for (Map.Entry entry : node2part.entrySet()) { + GridDhtPartitionState state = entry.getValue().get(p); + ClusterNode n = ctx.discovery().node(entry.getKey()); + + if (n != null && state != null && (state == MOVING || state == OWNING || state == RENTING) + && !nodes.contains(n) && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) + nodes.add(n); + } } return nodes; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java index 9670f8a4ea5cf..c19d486a489b4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java @@ -174,4 +174,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite @Override public String toString() { return S.toString(CacheVersionedValue.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java index f594e2bd353f6..a607a9afafbc6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java @@ -360,74 +360,74 @@ public long accessTtl() { } switch (writer.state()) { - case 3: - if (!writer.writeLong("accessTtl", accessTtl)) - return false; - - writer.incrementState(); - case 4: - if (!writer.writeLong("createTtl", createTtl)) + if (!writer.writeLong("accessTtl", accessTtl)) return false; writer.incrementState(); case 5: - if (!writer.writeByte("flags", flags)) + if (!writer.writeLong("createTtl", createTtl)) return false; writer.incrementState(); case 6: - if (!writer.writeIgniteUuid("futId", futId)) + if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); case 7: - if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) + if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); case 8: - if (!writer.writeIgniteUuid("miniId", miniId)) + if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 9: - if (!writer.writeCollection("readersFlags", readersFlags, MessageCollectionItemType.BOOLEAN)) + if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); case 10: - if (!writer.writeUuid("subjId", subjId)) + if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); case 11: - if (!writer.writeInt("taskNameHash", taskNameHash)) + if (!writer.writeCollection("readersFlags", readersFlags, MessageCollectionItemType.BOOLEAN)) return false; writer.incrementState(); case 12: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); case 13: - if (!writer.writeMessage("ver", ver)) + if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); case 14: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) + return false; + + writer.incrementState(); + + case 15: + if (!writer.writeMessage("ver", ver)) return false; writer.incrementState(); @@ -448,7 +448,7 @@ public long accessTtl() { return false; switch (reader.state()) { - case 3: + case 4: accessTtl = reader.readLong("accessTtl"); if (!reader.isLastRead()) @@ -456,7 +456,7 @@ public long accessTtl() { reader.incrementState(); - case 4: + case 5: createTtl = reader.readLong("createTtl"); if (!reader.isLastRead()) @@ -464,7 +464,7 @@ public long accessTtl() { reader.incrementState(); - case 5: + case 6: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -472,7 +472,7 @@ public long accessTtl() { reader.incrementState(); - case 6: + case 7: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -480,7 +480,7 @@ public long accessTtl() { reader.incrementState(); - case 7: + case 8: keys = reader.readCollection("keys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -488,7 +488,7 @@ public long accessTtl() { reader.incrementState(); - case 8: + case 9: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -496,48 +496,48 @@ public long accessTtl() { reader.incrementState(); - case 9: - readersFlags = reader.readCollection("readersFlags", MessageCollectionItemType.BOOLEAN); + case 10: + mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 10: - subjId = reader.readUuid("subjId"); + case 11: + readersFlags = reader.readCollection("readersFlags", MessageCollectionItemType.BOOLEAN); if (!reader.isLastRead()) return false; reader.incrementState(); - case 11: - taskNameHash = reader.readInt("taskNameHash"); + case 12: + subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 12: - topVer = reader.readMessage("topVer"); + case 13: + taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 13: - ver = reader.readMessage("ver"); + case 14: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 14: - mvccSnapshot = reader.readMessage("mvccSnapshot"); + case 15: + ver = reader.readMessage("ver"); if (!reader.isLastRead()) return false; @@ -556,7 +556,7 @@ public long accessTtl() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 15; + return 16; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java index b4e4424862c54..578c46b6ac34a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java @@ -228,43 +228,43 @@ public void error(IgniteCheckedException err) { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeCollection("entries", entries, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeCollection("invalidParts", invalidParts, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeIgniteUuid("miniId", miniId)) return false; writer.incrementState(); - case 8: - if (!writer.writeMessage("topVer", topVer)) + case 9: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeMessage("ver", ver)) return false; @@ -286,7 +286,7 @@ public void error(IgniteCheckedException err) { return false; switch (reader.state()) { - case 3: + case 4: entries = reader.readCollection("entries", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -294,7 +294,7 @@ public void error(IgniteCheckedException err) { reader.incrementState(); - case 4: + case 5: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -302,7 +302,7 @@ public void error(IgniteCheckedException err) { reader.incrementState(); - case 5: + case 6: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -310,7 +310,7 @@ public void error(IgniteCheckedException err) { reader.incrementState(); - case 6: + case 7: invalidParts = reader.readCollection("invalidParts", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -318,7 +318,7 @@ public void error(IgniteCheckedException err) { reader.incrementState(); - case 7: + case 8: miniId = reader.readIgniteUuid("miniId"); if (!reader.isLastRead()) @@ -326,15 +326,15 @@ public void error(IgniteCheckedException err) { reader.incrementState(); - case 8: - topVer = reader.readMessage("topVer"); + case 9: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 9: + case 10: ver = reader.readMessage("ver"); if (!reader.isLastRead()) @@ -354,7 +354,7 @@ public void error(IgniteCheckedException err) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java index f736cae61848c..0712e2234f6cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java @@ -363,56 +363,56 @@ public long accessTtl() { } switch (writer.state()) { - case 20: + case 21: if (!writer.writeLong("accessTtl", accessTtl)) return false; writer.incrementState(); - case 21: + case 22: if (!writer.writeLong("createTtl", createTtl)) return false; writer.incrementState(); - case 22: + case 23: if (!writer.writeObjectArray("dhtVers", dhtVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 23: + case 24: if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 24: + case 25: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 25: + case 26: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 26: + case 27: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 27: + case 28: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 28: - if (!writer.writeMessage("topVer", topVer)) + case 29: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -433,7 +433,7 @@ public long accessTtl() { return false; switch (reader.state()) { - case 20: + case 21: accessTtl = reader.readLong("accessTtl"); if (!reader.isLastRead()) @@ -441,7 +441,7 @@ public long accessTtl() { reader.incrementState(); - case 21: + case 22: createTtl = reader.readLong("createTtl"); if (!reader.isLastRead()) @@ -449,7 +449,7 @@ public long accessTtl() { reader.incrementState(); - case 22: + case 23: dhtVers = reader.readObjectArray("dhtVers", MessageCollectionItemType.MSG, GridCacheVersion.class); if (!reader.isLastRead()) @@ -457,7 +457,7 @@ public long accessTtl() { reader.incrementState(); - case 23: + case 24: filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class); if (!reader.isLastRead()) @@ -465,7 +465,7 @@ public long accessTtl() { reader.incrementState(); - case 24: + case 25: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -473,7 +473,7 @@ public long accessTtl() { reader.incrementState(); - case 25: + case 26: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -481,7 +481,7 @@ public long accessTtl() { reader.incrementState(); - case 26: + case 27: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -489,7 +489,7 @@ public long accessTtl() { reader.incrementState(); - case 27: + case 28: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -497,8 +497,8 @@ public long accessTtl() { reader.incrementState(); - case 28: - topVer = reader.readMessage("topVer"); + case 29: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -517,7 +517,7 @@ public long accessTtl() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 29; + return 30; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java index e88f0a07e0755..b6c6d8c903c49 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java @@ -208,37 +208,37 @@ public void addValueBytes( } switch (writer.state()) { - case 10: - if (!writer.writeMessage("clientRemapVer", clientRemapVer)) + case 11: + if (!writer.writeAffinityTopologyVersion("clientRemapVer", clientRemapVer)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeObjectArray("dhtVers", dhtVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeBooleanArray("filterRes", filterRes)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeObjectArray("mappedVers", mappedVers, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG)) return false; @@ -260,15 +260,15 @@ public void addValueBytes( return false; switch (reader.state()) { - case 10: - clientRemapVer = reader.readMessage("clientRemapVer"); + case 11: + clientRemapVer = reader.readAffinityTopologyVersion("clientRemapVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 11: + case 12: dhtVers = reader.readObjectArray("dhtVers", MessageCollectionItemType.MSG, GridCacheVersion.class); if (!reader.isLastRead()) @@ -276,7 +276,7 @@ public void addValueBytes( reader.incrementState(); - case 12: + case 13: filterRes = reader.readBooleanArray("filterRes"); if (!reader.isLastRead()) @@ -284,7 +284,7 @@ public void addValueBytes( reader.incrementState(); - case 13: + case 14: mappedVers = reader.readObjectArray("mappedVers", MessageCollectionItemType.MSG, GridCacheVersion.class); if (!reader.isLastRead()) @@ -292,7 +292,7 @@ public void addValueBytes( reader.incrementState(); - case 14: + case 15: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -300,7 +300,7 @@ public void addValueBytes( reader.incrementState(); - case 15: + case 16: pending = reader.readCollection("pending", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -320,7 +320,7 @@ public void addValueBytes( /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 16; + return 17; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java index cf885e27111e3..44addb053243c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java @@ -296,7 +296,7 @@ public boolean recovery() { return false; switch (reader.state()) { - case 3: + case 4: accessTtl = reader.readLong("accessTtl"); if (!reader.isLastRead()) @@ -304,7 +304,7 @@ public boolean recovery() { reader.incrementState(); - case 4: + case 5: createTtl = reader.readLong("createTtl"); if (!reader.isLastRead()) @@ -312,7 +312,7 @@ public boolean recovery() { reader.incrementState(); - case 5: + case 6: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -320,7 +320,7 @@ public boolean recovery() { reader.incrementState(); - case 6: + case 7: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -328,7 +328,7 @@ public boolean recovery() { reader.incrementState(); - case 7: + case 8: key = reader.readMessage("key"); if (!reader.isLastRead()) @@ -336,32 +336,32 @@ public boolean recovery() { reader.incrementState(); - case 8: - subjId = reader.readUuid("subjId"); + case 9: + mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 9: - taskNameHash = reader.readInt("taskNameHash"); + case 10: + subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 10: - topVer = reader.readMessage("topVer"); + case 11: + taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 11: - mvccSnapshot = reader.readMessage("mvccSnapshot"); + case 12: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -388,56 +388,56 @@ public boolean recovery() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeLong("accessTtl", accessTtl)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeLong("createTtl", createTtl)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeMessage("key", key)) return false; writer.incrementState(); - case 8: - if (!writer.writeUuid("subjId", subjId)) + case 9: + if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); - case 9: - if (!writer.writeInt("taskNameHash", taskNameHash)) + case 10: + if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 10: - if (!writer.writeMessage("topVer", topVer)) + case 11: + if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 11: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) + case 12: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -459,7 +459,7 @@ public boolean recovery() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 12; + return 13; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java index 2cb75c253e290..584cec2eb1cb1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java @@ -206,32 +206,32 @@ else if (res instanceof GridCacheEntryInfo) } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeMessage("res", res)) return false; writer.incrementState(); - case 7: - if (!writer.writeMessage("topVer", topVer)) + case 8: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -252,7 +252,7 @@ else if (res instanceof GridCacheEntryInfo) return false; switch (reader.state()) { - case 3: + case 4: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -260,7 +260,7 @@ else if (res instanceof GridCacheEntryInfo) reader.incrementState(); - case 4: + case 5: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -268,7 +268,7 @@ else if (res instanceof GridCacheEntryInfo) reader.incrementState(); - case 5: + case 6: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -276,7 +276,7 @@ else if (res instanceof GridCacheEntryInfo) reader.incrementState(); - case 6: + case 7: res = reader.readMessage("res"); if (!reader.isLastRead()) @@ -284,8 +284,8 @@ else if (res instanceof GridCacheEntryInfo) reader.incrementState(); - case 7: - topVer = reader.readMessage("topVer"); + case 8: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -309,7 +309,7 @@ else if (res instanceof GridCacheEntryInfo) /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 8; + return 9; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java index e71de896a185b..dd868618c005e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java @@ -388,97 +388,97 @@ public CacheEntryPredicate filter() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeBoolean("clientFirst", clientFirst)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeMessage("filter", filter)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeObjectArray("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeMessage("lockVer", lockVer)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeBoolean("needRes", needRes)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeByte("op", op != null ? (byte)op.ordinal() : -1)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeLong("threadId", threadId)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 16: - if (!writer.writeMessage("topVer", topVer)) + case 17: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeLong("txTimeout", txTimeout)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeObjectArray("values", values, MessageCollectionItemType.MSG)) return false; @@ -500,7 +500,7 @@ public CacheEntryPredicate filter() { return false; switch (reader.state()) { - case 3: + case 4: clientFirst = reader.readBoolean("clientFirst"); if (!reader.isLastRead()) @@ -508,7 +508,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 4: + case 5: filter = reader.readMessage("filter"); if (!reader.isLastRead()) @@ -516,7 +516,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 5: + case 6: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -524,7 +524,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 6: + case 7: keys = reader.readObjectArray("keys", MessageCollectionItemType.MSG, KeyCacheObject.class); if (!reader.isLastRead()) @@ -532,7 +532,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 7: + case 8: lockVer = reader.readMessage("lockVer"); if (!reader.isLastRead()) @@ -540,7 +540,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 8: + case 9: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -548,7 +548,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 9: + case 10: mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) @@ -556,7 +556,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 10: + case 11: needRes = reader.readBoolean("needRes"); if (!reader.isLastRead()) @@ -564,7 +564,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 11: + case 12: byte opOrd; opOrd = reader.readByte("op"); @@ -576,7 +576,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 12: + case 13: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -584,7 +584,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 13: + case 14: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -592,7 +592,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 14: + case 15: threadId = reader.readLong("threadId"); if (!reader.isLastRead()) @@ -600,7 +600,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 15: + case 16: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -608,15 +608,15 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 16: - topVer = reader.readMessage("topVer"); + case 17: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 17: + case 18: txTimeout = reader.readLong("txTimeout"); if (!reader.isLastRead()) @@ -624,7 +624,7 @@ public CacheEntryPredicate filter() { reader.incrementState(); - case 18: + case 19: values = reader.readObjectArray("values", MessageCollectionItemType.MSG, Message.class); if (!reader.isLastRead()) @@ -639,7 +639,7 @@ public CacheEntryPredicate filter() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 19; + return 20; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java index 78333a4e65ae7..cf00186eb1213 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java @@ -190,7 +190,7 @@ public Collection newDhtNodes() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 11; + return 12; } /** {@inheritDoc} */ @@ -208,49 +208,49 @@ public Collection newDhtNodes() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeIgniteUuid("dhtFutId", dhtFutId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeMessage("dhtVer", dhtVer)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeMessage("lockVer", lockVer)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeCollection("newDhtNodes", newDhtNodes, MessageCollectionItemType.UUID)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeMessage("res", res)) return false; @@ -272,7 +272,7 @@ public Collection newDhtNodes() { return false; switch (reader.state()) { - case 3: + case 4: dhtFutId = reader.readIgniteUuid("dhtFutId"); if (!reader.isLastRead()) @@ -280,7 +280,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 4: + case 5: dhtVer = reader.readMessage("dhtVer"); if (!reader.isLastRead()) @@ -288,7 +288,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 5: + case 6: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -296,7 +296,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 6: + case 7: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -304,7 +304,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 7: + case 8: lockVer = reader.readMessage("lockVer"); if (!reader.isLastRead()) @@ -312,7 +312,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 8: + case 9: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -320,7 +320,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 9: + case 10: newDhtNodes = reader.readCollection("newDhtNodes", MessageCollectionItemType.UUID); if (!reader.isLastRead()) @@ -328,7 +328,7 @@ public Collection newDhtNodes() { reader.incrementState(); - case 10: + case 11: res = reader.readMessage("res"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java index 6b5aa90e3f5f3..91079dff6c13f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java @@ -186,13 +186,13 @@ public void miniId(int miniId) { } switch (writer.state()) { - case 21: + case 22: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 22: + case 23: if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; @@ -214,7 +214,7 @@ public void miniId(int miniId) { return false; switch (reader.state()) { - case 21: + case 22: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -222,7 +222,7 @@ public void miniId(int miniId) { reader.incrementState(); - case 22: + case 23: mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) @@ -242,7 +242,7 @@ public void miniId(int miniId) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 23; + return 24; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java index a1a2b5712fcdd..e3dcbf832bd14 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java @@ -133,19 +133,19 @@ public long threadId() { } switch (writer.state()) { - case 6: + case 7: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeLong("nearThreadId", nearThreadId)) return false; @@ -167,7 +167,7 @@ public long threadId() { return false; switch (reader.state()) { - case 6: + case 7: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -175,7 +175,7 @@ public long threadId() { reader.incrementState(); - case 7: + case 8: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -183,7 +183,7 @@ public long threadId() { reader.incrementState(); - case 8: + case 9: nearThreadId = reader.readLong("nearThreadId"); if (!reader.isLastRead()) @@ -203,7 +203,7 @@ public long threadId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 9; + return 10; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java index 55c809d6f189f..cb17e6a9e7373 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java @@ -318,38 +318,38 @@ private boolean isFlag(int mask) { } switch (writer.state()) { - case 20: + case 21: if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); - case 21: + case 22: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 22: + case 23: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 23: + case 24: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 24: + case 25: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 25: - if (!writer.writeMessage("topVer", topVer)) + case 26: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -370,7 +370,7 @@ private boolean isFlag(int mask) { return false; switch (reader.state()) { - case 20: + case 21: flags = reader.readByte("flags"); if (!reader.isLastRead()) @@ -378,7 +378,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 21: + case 22: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -386,7 +386,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 22: + case 23: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -394,7 +394,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 23: + case 24: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -402,7 +402,7 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 24: + case 25: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -410,8 +410,8 @@ private boolean isFlag(int mask) { reader.incrementState(); - case 25: - topVer = reader.readMessage("topVer"); + case 26: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; @@ -430,7 +430,7 @@ private boolean isFlag(int mask) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 26; + return 27; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java index e9865df5e3e3b..67f7116b7a909 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java @@ -376,68 +376,68 @@ public boolean hasOwnedValue(IgniteTxKey key) { } switch (writer.state()) { - case 10: - if (!writer.writeMessage("clientRemapVer", clientRemapVer)) - return false; - - writer.incrementState(); - case 11: - if (!writer.writeMessage("dhtVer", dhtVer)) + if (!writer.writeAffinityTopologyVersion("clientRemapVer", clientRemapVer)) return false; writer.incrementState(); case 12: - if (!writer.writeCollection("filterFailedKeys", filterFailedKeys, MessageCollectionItemType.MSG)) + if (!writer.writeMessage("dhtVer", dhtVer)) return false; writer.incrementState(); case 13: - if (!writer.writeIgniteUuid("futId", futId)) + if (!writer.writeCollection("filterFailedKeys", filterFailedKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 14: - if (!writer.writeInt("miniId", miniId)) + if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); case 15: - if (!writer.writeCollection("ownedValKeys", ownedValKeys, MessageCollectionItemType.MSG)) + if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); case 16: - if (!writer.writeCollection("ownedValVals", ownedValVals, MessageCollectionItemType.MSG)) + if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); case 17: - if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG)) + if (!writer.writeCollection("ownedValKeys", ownedValKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 18: - if (!writer.writeMessage("retVal", retVal)) + if (!writer.writeCollection("ownedValVals", ownedValVals, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 19: - if (!writer.writeMessage("writeVer", writeVer)) + if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG)) return false; writer.incrementState(); case 20: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) + if (!writer.writeMessage("retVal", retVal)) + return false; + + writer.incrementState(); + + case 21: + if (!writer.writeMessage("writeVer", writeVer)) return false; writer.incrementState(); @@ -458,16 +458,8 @@ public boolean hasOwnedValue(IgniteTxKey key) { return false; switch (reader.state()) { - case 10: - clientRemapVer = reader.readMessage("clientRemapVer"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - case 11: - dhtVer = reader.readMessage("dhtVer"); + clientRemapVer = reader.readAffinityTopologyVersion("clientRemapVer"); if (!reader.isLastRead()) return false; @@ -475,7 +467,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 12: - filterFailedKeys = reader.readCollection("filterFailedKeys", MessageCollectionItemType.MSG); + dhtVer = reader.readMessage("dhtVer"); if (!reader.isLastRead()) return false; @@ -483,7 +475,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 13: - futId = reader.readIgniteUuid("futId"); + filterFailedKeys = reader.readCollection("filterFailedKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -491,7 +483,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 14: - miniId = reader.readInt("miniId"); + futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) return false; @@ -499,7 +491,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 15: - ownedValKeys = reader.readCollection("ownedValKeys", MessageCollectionItemType.MSG); + miniId = reader.readInt("miniId"); if (!reader.isLastRead()) return false; @@ -507,7 +499,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 16: - ownedValVals = reader.readCollection("ownedValVals", MessageCollectionItemType.MSG); + mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) return false; @@ -515,7 +507,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 17: - pending = reader.readCollection("pending", MessageCollectionItemType.MSG); + ownedValKeys = reader.readCollection("ownedValKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -523,7 +515,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 18: - retVal = reader.readMessage("retVal"); + ownedValVals = reader.readCollection("ownedValVals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -531,7 +523,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 19: - writeVer = reader.readMessage("writeVer"); + pending = reader.readCollection("pending", MessageCollectionItemType.MSG); if (!reader.isLastRead()) return false; @@ -539,7 +531,15 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 20: - mvccSnapshot = reader.readMessage("mvccSnapshot"); + retVal = reader.readMessage("retVal"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 21: + writeVer = reader.readMessage("writeVer"); if (!reader.isLastRead()) return false; @@ -558,7 +558,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 21; + return 22; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistRequest.java index 472937be41569..3b22afb7734b7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistRequest.java @@ -297,7 +297,7 @@ public boolean firstClientRequest() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 21; + return 22; } /** {@inheritDoc} */ @@ -331,109 +331,109 @@ public boolean firstClientRequest() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeIntArray("cacheIds", cacheIds)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeBoolean("clientFirst", clientFirst)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeInt("flags", flags)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeMessage("lockVer", lockVer)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeInt("pageSize", pageSize)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeByteArray("paramsBytes", paramsBytes)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeIntArray("parts", parts)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeString("qry", qry)) return false; writer.incrementState(); - case 14: + case 15: if (!writer.writeString("schema", schema)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 17: + case 18: if (!writer.writeLong("threadId", threadId)) return false; writer.incrementState(); - case 18: + case 19: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 19: - if (!writer.writeMessage("topVer", topVer)) + case 20: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 20: + case 21: if (!writer.writeLong("txTimeout", txTimeout)) return false; @@ -455,7 +455,7 @@ public boolean firstClientRequest() { return false; switch (reader.state()) { - case 3: + case 4: cacheIds = reader.readIntArray("cacheIds"); if (!reader.isLastRead()) @@ -463,7 +463,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 4: + case 5: clientFirst = reader.readBoolean("clientFirst"); if (!reader.isLastRead()) @@ -471,7 +471,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 5: + case 6: flags = reader.readInt("flags"); if (!reader.isLastRead()) @@ -479,7 +479,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 6: + case 7: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -487,7 +487,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 7: + case 8: lockVer = reader.readMessage("lockVer"); if (!reader.isLastRead()) @@ -495,7 +495,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 8: + case 9: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -503,7 +503,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 9: + case 10: mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) @@ -511,7 +511,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 10: + case 11: pageSize = reader.readInt("pageSize"); if (!reader.isLastRead()) @@ -519,7 +519,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 11: + case 12: paramsBytes = reader.readByteArray("paramsBytes"); if (!reader.isLastRead()) @@ -527,7 +527,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 12: + case 13: parts = reader.readIntArray("parts"); if (!reader.isLastRead()) @@ -535,7 +535,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 13: + case 14: qry = reader.readString("qry"); if (!reader.isLastRead()) @@ -543,7 +543,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 14: + case 15: schema = reader.readString("schema"); if (!reader.isLastRead()) @@ -551,7 +551,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 15: + case 16: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -559,7 +559,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 16: + case 17: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -567,7 +567,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 17: + case 18: threadId = reader.readLong("threadId"); if (!reader.isLastRead()) @@ -575,7 +575,7 @@ public boolean firstClientRequest() { reader.incrementState(); - case 18: + case 19: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -583,15 +583,15 @@ public boolean firstClientRequest() { reader.incrementState(); - case 19: - topVer = reader.readMessage("topVer"); + case 20: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 20: + case 21: txTimeout = reader.readLong("txTimeout"); if (!reader.isLastRead()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java index d628de1d56311..2715f89b408f3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java @@ -166,7 +166,7 @@ public boolean removeMapping() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 10; + return 11; } /** {@inheritDoc} */ @@ -184,47 +184,48 @@ public boolean removeMapping() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeMessage("lockVer", lockVer)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 7: - if (!writer.writeBoolean("removeMapping", removeMapping)) + case 8: + if (!writer.writeCollection("newDhtNodes", newDhtNodes, MessageCollectionItemType.UUID)) return false; writer.incrementState(); - case 8: - if (!writer.writeLong("res", res)) + case 9: + if (!writer.writeBoolean("removeMapping", removeMapping)) return false; writer.incrementState(); - case 9: - if (!writer.writeCollection("newDhtNodes", newDhtNodes, MessageCollectionItemType.UUID)) + case 10: + if (!writer.writeLong("res", res)) return false; writer.incrementState(); + } return true; @@ -241,7 +242,7 @@ public boolean removeMapping() { return false; switch (reader.state()) { - case 3: + case 4: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -249,7 +250,7 @@ public boolean removeMapping() { reader.incrementState(); - case 4: + case 5: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -257,7 +258,7 @@ public boolean removeMapping() { reader.incrementState(); - case 5: + case 6: lockVer = reader.readMessage("lockVer"); if (!reader.isLastRead()) @@ -265,7 +266,7 @@ public boolean removeMapping() { reader.incrementState(); - case 6: + case 7: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -273,29 +274,30 @@ public boolean removeMapping() { reader.incrementState(); - case 7: - removeMapping = reader.readBoolean("removeMapping"); + case 8: + newDhtNodes = reader.readCollection("newDhtNodes", MessageCollectionItemType.UUID); if (!reader.isLastRead()) return false; reader.incrementState(); - case 8: - res = reader.readLong("res"); + case 9: + removeMapping = reader.readBoolean("removeMapping"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 9: - newDhtNodes = reader.readCollection("newDhtNodes", MessageCollectionItemType.UUID); + case 10: + res = reader.readLong("res"); if (!reader.isLastRead()) return false; reader.incrementState(); + } return reader.afterMessageRead(GridNearTxQueryEnlistResponse.class); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistRequest.java index f350d502b9b7d..8e10a7edb3494 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistRequest.java @@ -335,85 +335,85 @@ public EnlistOperation operation() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeBoolean("clientFirst", clientFirst)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeObjectArray("keys", keys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeMessage("lockVer", lockVer)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeInt("miniId", miniId)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); - case 9: + case 10: if (!writer.writeByte("op", op != null ? (byte)op.ordinal() : -1)) return false; writer.incrementState(); - case 10: + case 11: if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeInt("taskNameHash", taskNameHash)) return false; writer.incrementState(); - case 12: + case 13: if (!writer.writeLong("threadId", threadId)) return false; writer.incrementState(); - case 13: + case 14: if (!writer.writeLong("timeout", timeout)) return false; writer.incrementState(); - case 14: - if (!writer.writeMessage("topVer", topVer)) + case 15: + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); - case 15: + case 16: if (!writer.writeLong("txTimeout", txTimeout)) return false; writer.incrementState(); - case 16: + case 17: if (!writer.writeObjectArray("values", values, MessageCollectionItemType.MSG)) return false; @@ -435,7 +435,7 @@ public EnlistOperation operation() { return false; switch (reader.state()) { - case 3: + case 4: clientFirst = reader.readBoolean("clientFirst"); if (!reader.isLastRead()) @@ -443,7 +443,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 4: + case 5: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -451,7 +451,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 5: + case 6: keys = reader.readObjectArray("keys", MessageCollectionItemType.MSG, KeyCacheObject.class); if (!reader.isLastRead()) @@ -459,7 +459,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 6: + case 7: lockVer = reader.readMessage("lockVer"); if (!reader.isLastRead()) @@ -467,7 +467,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 7: + case 8: miniId = reader.readInt("miniId"); if (!reader.isLastRead()) @@ -475,7 +475,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 8: + case 9: mvccSnapshot = reader.readMessage("mvccSnapshot"); if (!reader.isLastRead()) @@ -483,7 +483,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 9: + case 10: byte opOrd; opOrd = reader.readByte("op"); @@ -495,7 +495,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 10: + case 11: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -503,7 +503,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 11: + case 12: taskNameHash = reader.readInt("taskNameHash"); if (!reader.isLastRead()) @@ -511,7 +511,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 12: + case 13: threadId = reader.readLong("threadId"); if (!reader.isLastRead()) @@ -519,7 +519,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 13: + case 14: timeout = reader.readLong("timeout"); if (!reader.isLastRead()) @@ -527,15 +527,15 @@ public EnlistOperation operation() { reader.incrementState(); - case 14: - topVer = reader.readMessage("topVer"); + case 15: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 15: + case 16: txTimeout = reader.readLong("txTimeout"); if (!reader.isLastRead()) @@ -543,7 +543,7 @@ public EnlistOperation operation() { reader.incrementState(); - case 16: + case 17: values = reader.readObjectArray("values", MessageCollectionItemType.MSG, CacheObject.class); if (!reader.isLastRead()) @@ -558,7 +558,7 @@ public EnlistOperation operation() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 17; + return 18; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java index 48c63bc3c942c..2a0c63242f867 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java @@ -102,7 +102,7 @@ public IgniteUuid dhtFutureId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 12; + return 13; } /** {@inheritDoc} */ @@ -120,17 +120,18 @@ public IgniteUuid dhtFutureId() { } switch (writer.state()) { - case 10: + case 11: if (!writer.writeIgniteUuid("dhtFutId", dhtFutId)) return false; writer.incrementState(); - case 11: + case 12: if (!writer.writeMessage("dhtVer", dhtVer)) return false; writer.incrementState(); + } return true; @@ -147,7 +148,7 @@ public IgniteUuid dhtFutureId() { return false; switch (reader.state()) { - case 10: + case 11: dhtFutId = reader.readIgniteUuid("dhtFutId"); if (!reader.isLastRead()) @@ -155,13 +156,14 @@ public IgniteUuid dhtFutureId() { reader.incrementState(); - case 11: + case 12: dhtVer = reader.readMessage("dhtVer"); if (!reader.isLastRead()) return false; reader.incrementState(); + } return reader.afterMessageRead(GridNearTxQueryResultsEnlistResponse.class); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java index 2b49889cadc27..b2ed241e91fd2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java @@ -85,7 +85,7 @@ public GridNearUnlockRequest(int cacheId, int keyCnt, boolean addDepInfo) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 8; + return 9; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccAckRequestTxAndQueryId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccAckRequestTxAndQueryId.java index 89f09db5d0b2f..f3b3150480af1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccAckRequestTxAndQueryId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccAckRequestTxAndQueryId.java @@ -76,6 +76,7 @@ public MvccAckRequestTxAndQueryId(long futId, long txCntr, long qryTrackerId) { return false; writer.incrementState(); + } return true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccSnapshotResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccSnapshotResponse.java index c5a849499e0df..73d3f9483981a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccSnapshotResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccSnapshotResponse.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.managers.communication.GridIoMessageFactory; import org.apache.ignite.internal.processors.cache.mvcc.MvccLongList; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; @@ -33,6 +34,7 @@ /** * */ +@IgniteCodeGeneratingFail public class MvccSnapshotResponse implements MvccMessage, MvccSnapshot, MvccLongList { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccTxSnapshotRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccTxSnapshotRequest.java index cd30eb85b88af..4cf6f65a44de4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccTxSnapshotRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/MvccTxSnapshotRequest.java @@ -81,6 +81,7 @@ public long futureId() { return false; writer.incrementState(); + } return true; @@ -101,6 +102,7 @@ public long futureId() { return false; reader.incrementState(); + } return reader.afterMessageRead(MvccTxSnapshotRequest.class); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastRequest.java index ffd9a67edb6d1..5c46ca66a280a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastRequest.java @@ -79,13 +79,13 @@ public IgniteUuid futId() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeIgniteUuid("futId", futId)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeCollection("updCntrs", updCntrs, MessageCollectionItemType.MSG)) return false; @@ -107,7 +107,7 @@ public IgniteUuid futId() { return false; switch (reader.state()) { - case 3: + case 4: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -115,7 +115,7 @@ public IgniteUuid futId() { reader.incrementState(); - case 4: + case 5: updCntrs = reader.readCollection("updCntrs", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -135,7 +135,7 @@ public IgniteUuid futId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 5; + return 6; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastResponse.java index 547539db69ba3..2f88cde5a6fbb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/msg/PartitionCountersNeighborcastResponse.java @@ -62,7 +62,7 @@ public IgniteUuid futId() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeIgniteUuid("futId", futId)) return false; @@ -84,7 +84,7 @@ public IgniteUuid futId() { return false; switch (reader.state()) { - case 3: + case 4: futId = reader.readIgniteUuid("futId"); if (!reader.isLastRead()) @@ -104,7 +104,7 @@ public IgniteUuid futId() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 4; + return 5; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java index b7205b6e17bbb..21c636334e121 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java @@ -513,128 +513,128 @@ public int taskHash() { } switch (writer.state()) { - case 3: - if (!writer.writeBoolean("all", all)) - return false; - - writer.incrementState(); - case 4: - if (!writer.writeByteArray("argsBytes", argsBytes)) + if (!writer.writeBoolean("all", all)) return false; writer.incrementState(); case 5: - if (!writer.writeString("cacheName", cacheName)) + if (!writer.writeByteArray("argsBytes", argsBytes)) return false; writer.incrementState(); case 6: - if (!writer.writeBoolean("cancel", cancel)) + if (!writer.writeString("cacheName", cacheName)) return false; writer.incrementState(); case 7: - if (!writer.writeString("clause", clause)) + if (!writer.writeBoolean("cancel", cancel)) return false; writer.incrementState(); case 8: - if (!writer.writeString("clsName", clsName)) + if (!writer.writeString("clause", clause)) return false; writer.incrementState(); case 9: - if (!writer.writeBoolean("fields", fields)) + if (!writer.writeString("clsName", clsName)) return false; writer.incrementState(); case 10: - if (!writer.writeLong("id", id)) + if (!writer.writeBoolean("fields", fields)) return false; writer.incrementState(); case 11: - if (!writer.writeBoolean("incBackups", incBackups)) + if (!writer.writeLong("id", id)) return false; writer.incrementState(); case 12: - if (!writer.writeBoolean("incMeta", incMeta)) + if (!writer.writeBoolean("incBackups", incBackups)) return false; writer.incrementState(); case 13: - if (!writer.writeBoolean("keepBinary", keepBinary)) + if (!writer.writeBoolean("incMeta", incMeta)) return false; writer.incrementState(); case 14: - if (!writer.writeByteArray("keyValFilterBytes", keyValFilterBytes)) + if (!writer.writeBoolean("keepBinary", keepBinary)) return false; writer.incrementState(); case 15: - if (!writer.writeInt("pageSize", pageSize)) + if (!writer.writeByteArray("keyValFilterBytes", keyValFilterBytes)) return false; writer.incrementState(); case 16: - if (!writer.writeInt("part", part)) + if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) return false; writer.incrementState(); case 17: - if (!writer.writeByteArray("rdcBytes", rdcBytes)) + if (!writer.writeInt("pageSize", pageSize)) return false; writer.incrementState(); case 18: - if (!writer.writeUuid("subjId", subjId)) + if (!writer.writeInt("part", part)) return false; writer.incrementState(); case 19: - if (!writer.writeInt("taskHash", taskHash)) + if (!writer.writeByteArray("rdcBytes", rdcBytes)) return false; writer.incrementState(); case 20: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeUuid("subjId", subjId)) return false; writer.incrementState(); case 21: - if (!writer.writeByteArray("transBytes", transBytes)) + if (!writer.writeInt("taskHash", taskHash)) return false; writer.incrementState(); case 22: - if (!writer.writeByte("type", type != null ? (byte)type.ordinal() : -1)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); case 23: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) + if (!writer.writeByteArray("transBytes", transBytes)) + return false; + + writer.incrementState(); + + case 24: + if (!writer.writeByte("type", type != null ? (byte)type.ordinal() : -1)) return false; writer.incrementState(); @@ -655,7 +655,7 @@ public int taskHash() { return false; switch (reader.state()) { - case 3: + case 4: all = reader.readBoolean("all"); if (!reader.isLastRead()) @@ -663,7 +663,7 @@ public int taskHash() { reader.incrementState(); - case 4: + case 5: argsBytes = reader.readByteArray("argsBytes"); if (!reader.isLastRead()) @@ -671,7 +671,7 @@ public int taskHash() { reader.incrementState(); - case 5: + case 6: cacheName = reader.readString("cacheName"); if (!reader.isLastRead()) @@ -679,7 +679,7 @@ public int taskHash() { reader.incrementState(); - case 6: + case 7: cancel = reader.readBoolean("cancel"); if (!reader.isLastRead()) @@ -687,7 +687,7 @@ public int taskHash() { reader.incrementState(); - case 7: + case 8: clause = reader.readString("clause"); if (!reader.isLastRead()) @@ -695,7 +695,7 @@ public int taskHash() { reader.incrementState(); - case 8: + case 9: clsName = reader.readString("clsName"); if (!reader.isLastRead()) @@ -703,7 +703,7 @@ public int taskHash() { reader.incrementState(); - case 9: + case 10: fields = reader.readBoolean("fields"); if (!reader.isLastRead()) @@ -711,7 +711,7 @@ public int taskHash() { reader.incrementState(); - case 10: + case 11: id = reader.readLong("id"); if (!reader.isLastRead()) @@ -719,7 +719,7 @@ public int taskHash() { reader.incrementState(); - case 11: + case 12: incBackups = reader.readBoolean("incBackups"); if (!reader.isLastRead()) @@ -727,7 +727,7 @@ public int taskHash() { reader.incrementState(); - case 12: + case 13: incMeta = reader.readBoolean("incMeta"); if (!reader.isLastRead()) @@ -735,7 +735,7 @@ public int taskHash() { reader.incrementState(); - case 13: + case 14: keepBinary = reader.readBoolean("keepBinary"); if (!reader.isLastRead()) @@ -743,7 +743,7 @@ public int taskHash() { reader.incrementState(); - case 14: + case 15: keyValFilterBytes = reader.readByteArray("keyValFilterBytes"); if (!reader.isLastRead()) @@ -751,7 +751,15 @@ public int taskHash() { reader.incrementState(); - case 15: + case 16: + mvccSnapshot = reader.readMessage("mvccSnapshot"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 17: pageSize = reader.readInt("pageSize"); if (!reader.isLastRead()) @@ -759,7 +767,7 @@ public int taskHash() { reader.incrementState(); - case 16: + case 18: part = reader.readInt("part"); if (!reader.isLastRead()) @@ -767,7 +775,7 @@ public int taskHash() { reader.incrementState(); - case 17: + case 19: rdcBytes = reader.readByteArray("rdcBytes"); if (!reader.isLastRead()) @@ -775,7 +783,7 @@ public int taskHash() { reader.incrementState(); - case 18: + case 20: subjId = reader.readUuid("subjId"); if (!reader.isLastRead()) @@ -783,7 +791,7 @@ public int taskHash() { reader.incrementState(); - case 19: + case 21: taskHash = reader.readInt("taskHash"); if (!reader.isLastRead()) @@ -791,15 +799,15 @@ public int taskHash() { reader.incrementState(); - case 20: - topVer = reader.readMessage("topVer"); + case 22: + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - case 21: + case 23: transBytes = reader.readByteArray("transBytes"); if (!reader.isLastRead()) @@ -807,7 +815,7 @@ public int taskHash() { reader.incrementState(); - case 22: + case 24: byte typeOrd; typeOrd = reader.readByte("type"); @@ -819,14 +827,6 @@ public int taskHash() { reader.incrementState(); - case 23: - mvccSnapshot = reader.readMessage("mvccSnapshot"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - } return reader.afterMessageRead(GridCacheQueryRequest.class); @@ -839,7 +839,7 @@ public int taskHash() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 24; + return 25; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java index 13e0915875b92..a1650be162c46 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java @@ -287,37 +287,37 @@ public boolean fields() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeCollection("dataBytes", dataBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeBoolean("fields", fields)) return false; writer.incrementState(); - case 6: + case 7: if (!writer.writeBoolean("finished", finished)) return false; writer.incrementState(); - case 7: + case 8: if (!writer.writeCollection("metaDataBytes", metaDataBytes, MessageCollectionItemType.BYTE_ARR)) return false; writer.incrementState(); - case 8: + case 9: if (!writer.writeLong("reqId", reqId)) return false; @@ -339,7 +339,7 @@ public boolean fields() { return false; switch (reader.state()) { - case 3: + case 4: dataBytes = reader.readCollection("dataBytes", MessageCollectionItemType.BYTE_ARR); if (!reader.isLastRead()) @@ -347,7 +347,7 @@ public boolean fields() { reader.incrementState(); - case 4: + case 5: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -355,7 +355,7 @@ public boolean fields() { reader.incrementState(); - case 5: + case 6: fields = reader.readBoolean("fields"); if (!reader.isLastRead()) @@ -363,7 +363,7 @@ public boolean fields() { reader.incrementState(); - case 6: + case 7: finished = reader.readBoolean("finished"); if (!reader.isLastRead()) @@ -371,7 +371,7 @@ public boolean fields() { reader.incrementState(); - case 7: + case 8: metaDataBytes = reader.readCollection("metaDataBytes", MessageCollectionItemType.BYTE_ARR); if (!reader.isLastRead()) @@ -379,7 +379,7 @@ public boolean fields() { reader.incrementState(); - case 8: + case 9: reqId = reader.readLong("reqId"); if (!reader.isLastRead()) @@ -399,7 +399,7 @@ public boolean fields() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 9; + return 10; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java index ef0157ee70caa..9ba9ad2d1cd28 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java @@ -90,15 +90,14 @@ Map updateCntrs() { } switch (writer.state()) { - case 3: + case 4: if (!writer.writeUuid("routineId", routineId)) return false; writer.incrementState(); - case 4: - if (!writer.writeMap("updateCntrs", updateCntrs, MessageCollectionItemType.INT, - MessageCollectionItemType.LONG)) + case 5: + if (!writer.writeMap("updateCntrs", updateCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG)) return false; writer.incrementState(); @@ -119,7 +118,7 @@ Map updateCntrs() { return false; switch (reader.state()) { - case 3: + case 4: routineId = reader.readUuid("routineId"); if (!reader.isLastRead()) @@ -127,9 +126,8 @@ Map updateCntrs() { reader.incrementState(); - case 4: - updateCntrs = reader.readMap("updateCntrs", MessageCollectionItemType.INT, - MessageCollectionItemType.LONG, false); + case 5: + updateCntrs = reader.readMap("updateCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false); if (!reader.isLastRead()) return false; @@ -153,7 +151,7 @@ Map updateCntrs() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 5; + return 6; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java index 88005d047933f..b05c275451ffe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java @@ -411,7 +411,7 @@ CacheObject oldValue() { writer.incrementState(); case 8: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -500,7 +500,7 @@ CacheObject oldValue() { reader.incrementState(); case 8: - topVer = reader.readMessage("topVer"); + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 09a3be4ab1359..3102854d0f6cb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -690,6 +690,11 @@ private boolean needRemap(AffinityTopologyVersion expVer, if (expVer.equals(curVer)) return false; + AffinityTopologyVersion lastAffChangedTopVer = ctx.exchange().lastAffinityChangedTopologyVersion(expVer); + + if (curVer.compareTo(expVer) <= 0 && curVer.compareTo(lastAffChangedTopVer) >= 0) + return false; + // TODO IGNITE-6754 check mvcc crd for mvcc enabled txs. for (IgniteTxEntry e : F.concat(false, req.reads(), req.writes())) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java index efcb48b4714c3..2c3892f564dde 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java @@ -191,4 +191,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite @Override public String toString() { return S.toString(IgniteTxKey.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java index 94fe00527a488..86109c8f323a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java @@ -149,13 +149,13 @@ public Collection txKeys() { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 3: + case 4: if (!writer.writeObjectArray("txKeysArr", txKeysArr, MessageCollectionItemType.MSG)) return false; @@ -177,7 +177,7 @@ public Collection txKeys() { return false; switch (reader.state()) { - case 2: + case 3: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -185,7 +185,7 @@ public Collection txKeys() { reader.incrementState(); - case 3: + case 4: txKeysArr = reader.readObjectArray("txKeysArr", MessageCollectionItemType.MSG, IgniteTxKey.class); if (!reader.isLastRead()) @@ -205,7 +205,7 @@ public Collection txKeys() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 4; + return 5; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java index a5c8f0917da8c..df5caa978b609 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java @@ -239,25 +239,25 @@ public void addKey(IgniteTxKey key) { } switch (writer.state()) { - case 2: + case 3: if (!writer.writeLong("futId", futId)) return false; writer.incrementState(); - case 3: + case 4: if (!writer.writeObjectArray("locksArr", locksArr, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 4: + case 5: if (!writer.writeObjectArray("nearTxKeysArr", nearTxKeysArr, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 5: + case 6: if (!writer.writeObjectArray("txKeysArr", txKeysArr, MessageCollectionItemType.MSG)) return false; @@ -279,7 +279,7 @@ public void addKey(IgniteTxKey key) { return false; switch (reader.state()) { - case 2: + case 3: futId = reader.readLong("futId"); if (!reader.isLastRead()) @@ -287,7 +287,7 @@ public void addKey(IgniteTxKey key) { reader.incrementState(); - case 3: + case 4: locksArr = reader.readObjectArray("locksArr", MessageCollectionItemType.MSG, TxLockList.class); if (!reader.isLastRead()) @@ -295,7 +295,7 @@ public void addKey(IgniteTxKey key) { reader.incrementState(); - case 4: + case 5: nearTxKeysArr = reader.readObjectArray("nearTxKeysArr", MessageCollectionItemType.MSG, IgniteTxKey.class); if (!reader.isLastRead()) @@ -303,7 +303,7 @@ public void addKey(IgniteTxKey key) { reader.incrementState(); - case 5: + case 6: txKeysArr = reader.readObjectArray("txKeysArr", MessageCollectionItemType.MSG, IgniteTxKey.class); if (!reader.isLastRead()) @@ -323,7 +323,7 @@ public void addKey(IgniteTxKey key) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 6; + return 7; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java index 586a043cc8d22..ceea0a2d9a5cd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java @@ -24,6 +24,7 @@ import java.nio.ByteBuffer; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; import org.apache.ignite.internal.processors.cache.CacheObjectValueContext; @@ -39,6 +40,7 @@ /** * Raw versioned entry. */ +@IgniteCodeGeneratingFail public class GridCacheRawVersionedEntry extends DataStreamerEntry implements GridCacheVersionedEntry, GridCacheVersionable, Externalizable { /** */ @@ -381,4 +383,4 @@ public void prepareDirectMarshal(CacheObjectContext ctx) throws IgniteCheckedExc "valBytesLen", valBytes != null ? valBytes.length : "n/a", "super", super.toString()); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java index ec04ac355e5a6..5e7cc9d3a4576 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java @@ -274,4 +274,4 @@ public void dataBytes(byte[] dataBytes) { @Override public String toString() { return S.toString(GridContinuousMessage.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java index f70ee9c486fa0..87313de15ebd2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java @@ -369,7 +369,7 @@ public int partition() { writer.incrementState(); case 13: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -508,7 +508,7 @@ public int partition() { reader.incrementState(); case 13: - topVer = reader.readMessage("topVer"); + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java index 4cb46e1582022..56f37c2cf0b98 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java @@ -169,4 +169,4 @@ public boolean forceLocalDeployment() { @Override public byte fieldsCount() { return 3; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java index f6bddb64fa366..e990cb3cd7303 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.hadoop.shuffle; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.hadoop.HadoopJobId; import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage; import org.apache.ignite.internal.util.tostring.GridToStringInclude; @@ -35,6 +36,7 @@ /** * Direct shuffle message. */ +@IgniteCodeGeneratingFail public class HadoopDirectShuffleMessage implements Message, HadoopMessage { /** */ private static final long serialVersionUID = 0L; @@ -268,4 +270,4 @@ public int dataLength() { @Override public String toString() { return S.toString(HadoopDirectShuffleMessage.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java index 07b8c2fd1635a..2660e281f20de 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.hadoop.HadoopJobId; import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage; import org.apache.ignite.internal.util.GridUnsafe; @@ -36,6 +37,7 @@ /** * Shuffle message. */ +@IgniteCodeGeneratingFail public class HadoopShuffleMessage implements Message, HadoopMessage { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java index 5ae3fed3e47e0..76a08f9877aab 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java @@ -195,4 +195,4 @@ public IgniteCheckedException error() { @Override public byte fieldsCount() { return 3; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java index 736525de1489a..98bf306a479df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java @@ -297,4 +297,4 @@ public boolean evictExclude() { @Override public String toString() { return S.toString(IgfsBlockKey.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java index 2ec54b2ed0663..5448c50b9352d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java @@ -176,4 +176,4 @@ public Map blocks() { @Override public byte fieldsCount() { return 3; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java index 412c45baf6daf..7f5daa617bdb7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java @@ -77,4 +77,4 @@ public void finishUnmarshal(Marshaller marsh, @Nullable ClassLoader ldr) throws @Override public byte fieldsCount() { return 0; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java index b5e96664b4634..2c2c98c82f18b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java @@ -190,4 +190,4 @@ public IgniteCheckedException error() { @Override public String toString() { return S.toString(IgfsDeleteMessage.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java index bb605b857641c..c64b627dac641 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java @@ -157,4 +157,4 @@ public Collection fragmentRanges() { @Override public byte fieldsCount() { return 2; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java index 76793bfeb6d0e..f7857dcfad154 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java @@ -118,4 +118,4 @@ public IgniteUuid fileId() { @Override public byte fieldsCount() { return 1; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java index 2b32084b7f3e6..2707de543002e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java @@ -149,4 +149,4 @@ public boolean response() { @Override public byte fieldsCount() { return 2; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java index 6b976c24179cf..7fdd5d6a57ceb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java @@ -220,7 +220,7 @@ public Collection plainRows() { writer.incrementState(); case 6: - if (!writer.writeMessage("retry", retry)) + if (!writer.writeAffinityTopologyVersion("retry", retry)) return false; writer.incrementState(); @@ -310,7 +310,7 @@ public Collection plainRows() { reader.incrementState(); case 6: - retry = reader.readMessage("retry"); + retry = reader.readAffinityTopologyVersion("retry"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java index 5f75e607d4173..9e81d5a84b432 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.query.schema.message; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageReader; @@ -30,6 +31,7 @@ /** * Schema operation status message. */ +@IgniteCodeGeneratingFail public class SchemaOperationStatusMessage implements Message { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java index b8b4edbf01bb7..f3ec0e8afacaf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java @@ -171,4 +171,4 @@ public void topic(String topic) { @Override public byte fieldsCount() { return 2; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java index b9bb27c067182..88b42f5e80a5c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java @@ -217,4 +217,4 @@ public void error(String err) { @Override public byte fieldsCount() { return 4; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java index 0200d7747002e..e9c7d016bdb4c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java @@ -491,4 +491,4 @@ public InputStream inputStream() { @Override public String toString() { return S.toString(GridByteArrayList.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java index ebe2e0765c787..c54584a78c4f1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java @@ -600,4 +600,4 @@ public GridIntIterator iterator() { } }; } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java index d1f20e634a03c..1496c375981f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.NoSuchElementException; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.SB; @@ -38,6 +39,7 @@ * Minimal list API to work with primitive longs. This list exists * to avoid boxing/unboxing when using standard list from Java. */ +@IgniteCodeGeneratingFail public class GridLongList implements Message, Externalizable { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java index 2134912e5a500..2340dd9d6afe6 100644 --- a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java +++ b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java @@ -84,7 +84,10 @@ public enum MessageCollectionItemType { IGNITE_UUID, /** Message. */ - MSG; + MSG, + + /** Topology version. */ + AFFINITY_TOPOLOGY_VERSION; /** Enum values. */ private static final MessageCollectionItemType[] VALS = values(); diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java index 050204288927d..6feee1a21eab4 100644 --- a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java +++ b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java @@ -23,6 +23,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.lang.IgniteUuid; /** @@ -229,6 +230,14 @@ public interface MessageReader { */ public IgniteUuid readIgniteUuid(String name); + /** + * Reads {@link AffinityTopologyVersion}. + * + * @param name Field name. + * @return {@link AffinityTopologyVersion}. + */ + public AffinityTopologyVersion readAffinityTopologyVersion(String name); + /** * Reads nested message. * diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java index 692955f39cf15..14d4417be3ee2 100644 --- a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java +++ b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Map; import java.util.UUID; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.lang.IgniteUuid; /** @@ -254,6 +255,15 @@ public interface MessageWriter { */ public boolean writeIgniteUuid(String name, IgniteUuid val); + /** + * Writes {@link AffinityTopologyVersion}. + * + * @param name Field name. + * @param val {@link AffinityTopologyVersion}. + * @return Whether value was fully written. + */ + public boolean writeAffinityTopologyVersion(String name, AffinityTopologyVersion val); + /** * Writes nested message. * diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java index 6ecb14572ee2b..5f028047981e8 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java @@ -118,4 +118,4 @@ int delta() { @Override public String toString() { return S.toString(JobStealingRequest.class, this); } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java index e3be9c99c7ed0..f845b0b5bbc19 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import java.util.UUID; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.Message; @@ -29,6 +30,7 @@ /** * Handshake message. */ +@IgniteCodeGeneratingFail public class HandshakeMessage implements Message { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java index f27a825740cae..220781371a69f 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import java.util.UUID; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; @@ -26,6 +27,7 @@ /** * Updated handshake message. */ +@IgniteCodeGeneratingFail public class HandshakeMessage2 extends HandshakeMessage { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java index 2c6aa300bd8be..12e2522ce4f05 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import java.util.UUID; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.Message; @@ -29,6 +30,7 @@ /** * Node ID message. */ +@IgniteCodeGeneratingFail public class NodeIdMessage implements Message { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java index eef2655a3eafa..95d43f6b868c5 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java @@ -18,6 +18,7 @@ package org.apache.ignite.spi.communication.tcp.messages; import java.nio.ByteBuffer; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageReader; @@ -27,6 +28,7 @@ /** * Recovery acknowledgment message. */ +@IgniteCodeGeneratingFail public class RecoveryLastReceivedMessage implements Message { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java new file mode 100644 index 0000000000000..45c6f25bf275a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.locks.Lock; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage; +import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * + */ +public class CacheNoAffinityExchangeTest extends GridCommonAbstractTest { + /** */ + private volatile boolean startClient; + + /** */ + private final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder().setShared(true); + + /** */ + private final TcpDiscoveryIpFinder CLIENT_IP_FINDER = new TcpDiscoveryVmIpFinder() + .setAddresses(Collections.singleton("127.0.0.1:47500")); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDiscoverySpi(new TestDiscoverySpi().setIpFinder(IP_FINDER)); + + if (startClient) { + cfg.setClientMode(true); + + // It is necessary to ensure that client always connects to grid(0). + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(CLIENT_IP_FINDER); + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * @throws Exception If failed. + */ + public void testNoAffinityChangeOnClientJoin() throws Exception { + Ignite ig = startGrids(4); + + ig.cluster().active(true); + + IgniteCache atomicCache = ig.createCache(new CacheConfiguration() + .setName("atomic").setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + IgniteCache txCache = ig.createCache(new CacheConfiguration() + .setName("tx").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + + assertTrue(GridTestUtils.waitForCondition(() -> + new AffinityTopologyVersion(4, 3).equals(grid(3).context().discovery().topologyVersionEx()), + 5_000)); + + TestDiscoverySpi discoSpi = (TestDiscoverySpi) grid(2).context().discovery().getInjectedDiscoverySpi(); + + CountDownLatch latch = new CountDownLatch(1); + + discoSpi.latch = latch; + + startClient = true; + + startGrid(4); + + assertTrue(GridTestUtils.waitForCondition(() -> + new AffinityTopologyVersion(5, 0).equals(grid(0).context().discovery().topologyVersionEx()) && + new AffinityTopologyVersion(5, 0).equals(grid(1).context().discovery().topologyVersionEx()) && + new AffinityTopologyVersion(4, 3).equals(grid(2).context().discovery().topologyVersionEx()) && + new AffinityTopologyVersion(4, 3).equals(grid(3).context().discovery().topologyVersionEx()), + 10_000)); + + for (int k = 0; k < 100; k++) { + atomicCache.put(k, k); + txCache.put(k, k); + + Lock lock = txCache.lock(k); + lock.lock(); + lock.unlock(); + } + + for (int k = 0; k < 100; k++) { + assertEquals(Integer.valueOf(k), atomicCache.get(k)); + assertEquals(Integer.valueOf(k), txCache.get(k)); + } + + assertEquals(new AffinityTopologyVersion(5, 0), grid(0).context().discovery().topologyVersionEx()); + assertEquals(new AffinityTopologyVersion(5, 0), grid(1).context().discovery().topologyVersionEx()); + assertEquals(new AffinityTopologyVersion(4, 3), grid(2).context().discovery().topologyVersionEx()); + assertEquals(new AffinityTopologyVersion(4, 3), grid(3).context().discovery().topologyVersionEx()); + + latch.countDown(); + } + + /** + * @throws Exception If failed. + */ + public void testNoAffinityChangeOnClientLeft() throws Exception { + Ignite ig = startGrids(4); + + ig.cluster().active(true); + + IgniteCache atomicCache = ig.createCache(new CacheConfiguration() + .setName("atomic").setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + IgniteCache txCache = ig.createCache(new CacheConfiguration() + .setName("tx").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + + assertTrue(GridTestUtils.waitForCondition(() -> + new AffinityTopologyVersion(4, 3).equals(grid(3).context().discovery().topologyVersionEx()), + 5_000)); + + startClient = true; + + startGrid(4); + + TestDiscoverySpi discoSpi = (TestDiscoverySpi)grid(2).context().discovery().getInjectedDiscoverySpi(); + + CountDownLatch latch = new CountDownLatch(1); + + discoSpi.latch = latch; + + stopGrid(4); + + assertTrue(GridTestUtils.waitForCondition(() -> + new AffinityTopologyVersion(6, 0).equals(grid(0).context().discovery().topologyVersionEx()) && + new AffinityTopologyVersion(6, 0).equals(grid(1).context().discovery().topologyVersionEx()) && + new AffinityTopologyVersion(5, 0).equals(grid(2).context().discovery().topologyVersionEx()) && + new AffinityTopologyVersion(5, 0).equals(grid(3).context().discovery().topologyVersionEx()), + 10_000)); + + for (int k = 0; k < 100; k++) { + atomicCache.put(k, k); + txCache.put(k, k); + + Lock lock = txCache.lock(k); + lock.lock(); + lock.unlock(); + } + + for (int k = 0; k < 100; k++) { + assertEquals(Integer.valueOf(k), atomicCache.get(k)); + assertEquals(Integer.valueOf(k), txCache.get(k)); + } + + assertEquals(new AffinityTopologyVersion(6, 0), grid(0).context().discovery().topologyVersionEx()); + assertEquals(new AffinityTopologyVersion(6, 0), grid(1).context().discovery().topologyVersionEx()); + assertEquals(new AffinityTopologyVersion(5, 0), grid(2).context().discovery().topologyVersionEx()); + assertEquals(new AffinityTopologyVersion(5, 0), grid(3).context().discovery().topologyVersionEx()); + + latch.countDown(); + } + + /** + * + */ + public static class TestDiscoverySpi extends TcpDiscoverySpi { + /** */ + private volatile CountDownLatch latch; + + /** {@inheritDoc} */ + @Override protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) { + if (msg instanceof TcpDiscoveryNodeAddFinishedMessage || msg instanceof TcpDiscoveryNodeLeftMessage || msg instanceof TcpDiscoveryNodeFailedMessage) { + CountDownLatch latch0 = latch; + + if (latch0 != null) + try { + latch0.await(); + } + catch (InterruptedException ex) { + throw new IgniteException(ex); + } + } + + super.startMessageProcess(msg); + } + } + +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java index 9e2286e43fe8c..349d806baa7d6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -846,8 +846,7 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws checkCaches1(SRVS + CLIENTS); // Wait for late affinity assignment to finish. - grid(0).context().cache().context().exchange().affinityReadyFuture( - new AffinityTopologyVersion(SRVS + CLIENTS, 1)).get(); + awaitPartitionMapExchange(); final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(SRVS + CLIENTS + 1, 1); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java index d95a7bf978aa3..49318f9719b23 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java @@ -457,6 +457,8 @@ private void concurrentStart(final boolean withClients) throws Exception { * @throws Exception If failed. */ public void testMergeServerAndClientJoin1() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10186"); + final IgniteEx srv0 = startGrid(0); mergeExchangeWaitVersion(srv0, 3); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index 1269d0d1bbb40..1697d32d25ad8 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.CacheNoAffinityExchangeTest; import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest; import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest; import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest; @@ -111,6 +112,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class); suite.addTestSuite(CacheTryLockMultithreadedTest.class); + suite.addTestSuite(CacheNoAffinityExchangeTest.class); + //suite.addTestSuite(CacheClientsConcurrentStartTest.class); //suite.addTestSuite(GridCacheRebalancingOrderingTest.class); //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java index 54f5f03f556b1..ca6343c9c7e4d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java @@ -93,6 +93,7 @@ public String table() { return false; writer.incrementState(); + } return true; @@ -121,6 +122,7 @@ public String table() { return false; reader.incrementState(); + } return reader.afterMessageRead(QueryTable.class); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java index 702488487d9c0..186104ad44b54 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.UUID; import org.apache.ignite.internal.GridDirectCollection; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; @@ -30,6 +31,7 @@ /** * Range request. */ +@IgniteCodeGeneratingFail public class GridH2IndexRangeRequest implements Message { /** */ private UUID originNodeId; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java index 4fe660c0677a1..18814bb3f71a7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.UUID; import org.apache.ignite.internal.GridDirectCollection; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; @@ -30,6 +31,7 @@ /** * Range response message. */ +@IgniteCodeGeneratingFail public class GridH2IndexRangeResponse implements Message { /** */ public static final byte STATUS_OK = 0; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java index 0bec66e56be42..cca366af7ce9f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.GridDirectMap; import org.apache.ignite.internal.GridDirectTransient; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; @@ -50,6 +51,7 @@ /** * Query request. */ +@IgniteCodeGeneratingFail public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable { /** */ private static final long serialVersionUID = 0L; @@ -518,7 +520,7 @@ public void txDetails(GridH2SelectForUpdateTxDetails txDetails) { writer.incrementState(); case 9: - if (!writer.writeMessage("topVer", topVer)) + if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; writer.incrementState(); @@ -633,7 +635,7 @@ public void txDetails(GridH2SelectForUpdateTxDetails txDetails) { reader.incrementState(); case 9: - topVer = reader.readMessage("topVer"); + topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) return false; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java index 18f88803bd218..fd5b5243a7be8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; @@ -28,6 +29,7 @@ /** * Abstract message wrapper for H2 values. */ +@IgniteCodeGeneratingFail public abstract class GridH2ValueMessage implements Message { /** * Gets H2 value. From 84f2abfef735220b69634ae9fdf21010f2303fda Mon Sep 17 00:00:00 2001 From: Max-Pudov Date: Mon, 19 Nov 2018 17:31:47 +0300 Subject: [PATCH 105/403] IGNITE-10098 .NET: Add missing TcpCommunicationSpi properties This closes #5379 (cherry picked from commit a198b3ff5799899690419e3c0b6306a1eaebd6e7) --- .../utils/PlatformConfigurationUtils.java | 12 +++ ...Apache.Ignite.Core.Tests.DotNetCore.csproj | 1 + .../Apache.Ignite.Core.Tests.csproj | 1 + .../TcpCommunicationSpiParityTest.cs | 84 ++++++++++++++++ .../IgniteConfigurationTest.cs | 16 ++- .../Communication/Tcp/TcpCommunicationSpi.cs | 99 +++++++++++++++++-- 6 files changed, 201 insertions(+), 12 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TcpCommunicationSpiParityTest.cs diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java index 915075afe7a34..8338a3b9849cd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java @@ -713,9 +713,11 @@ public static void readIgniteConfiguration(BinaryRawReaderEx in, IgniteConfigura TcpCommunicationSpi comm = new TcpCommunicationSpi(); comm.setAckSendThreshold(in.readInt()); + comm.setConnectionsPerNode(in.readInt()); comm.setConnectTimeout(in.readLong()); comm.setDirectBuffer(in.readBoolean()); comm.setDirectSendBuffer(in.readBoolean()); + comm.setFilterReachableAddresses(in.readBoolean()); comm.setIdleConnectionTimeout(in.readLong()); comm.setLocalAddress(in.readString()); comm.setLocalPort(in.readInt()); @@ -724,11 +726,15 @@ public static void readIgniteConfiguration(BinaryRawReaderEx in, IgniteConfigura comm.setMessageQueueLimit(in.readInt()); comm.setReconnectCount(in.readInt()); comm.setSelectorsCount(in.readInt()); + comm.setSelectorSpins(in.readLong()); + comm.setSharedMemoryPort(in.readInt()); comm.setSlowClientQueueLimit(in.readInt()); comm.setSocketReceiveBuffer(in.readInt()); comm.setSocketSendBuffer(in.readInt()); + comm.setSocketWriteTimeout(in.readLong()); comm.setTcpNoDelay(in.readBoolean()); comm.setUnacknowledgedMessagesBufferSize(in.readInt()); + comm.setUsePairedConnections(in.readBoolean()); cfg.setCommunicationSpi(comm); } @@ -1299,9 +1305,11 @@ public static void writeIgniteConfiguration(BinaryRawWriter w, IgniteConfigurati TcpCommunicationSpi tcp = (TcpCommunicationSpi) comm; w.writeInt(tcp.getAckSendThreshold()); + w.writeInt(tcp.getConnectionsPerNode()); w.writeLong(tcp.getConnectTimeout()); w.writeBoolean(tcp.isDirectBuffer()); w.writeBoolean(tcp.isDirectSendBuffer()); + w.writeBoolean(tcp.isFilterReachableAddresses()); w.writeLong(tcp.getIdleConnectionTimeout()); w.writeString(tcp.getLocalAddress()); w.writeInt(tcp.getLocalPort()); @@ -1310,11 +1318,15 @@ public static void writeIgniteConfiguration(BinaryRawWriter w, IgniteConfigurati w.writeInt(tcp.getMessageQueueLimit()); w.writeInt(tcp.getReconnectCount()); w.writeInt(tcp.getSelectorsCount()); + w.writeLong(tcp.getSelectorSpins()); + w.writeInt(tcp.getSharedMemoryPort()); w.writeInt(tcp.getSlowClientQueueLimit()); w.writeInt(tcp.getSocketReceiveBuffer()); w.writeInt(tcp.getSocketSendBuffer()); + w.writeLong(tcp.getSocketWriteTimeout()); w.writeBoolean(tcp.isTcpNoDelay()); w.writeInt(tcp.getUnacknowledgedMessagesBufferSize()); + w.writeBoolean(tcp.isUsePairedConnections()); } else w.writeBoolean(false); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Apache.Ignite.Core.Tests.DotNetCore.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Apache.Ignite.Core.Tests.DotNetCore.csproj index 6550a7fe6956b..2ec0bdb905951 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Apache.Ignite.Core.Tests.DotNetCore.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Apache.Ignite.Core.Tests.DotNetCore.csproj @@ -42,6 +42,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index aa58afc110e68..e2d4146bee5e5 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -94,6 +94,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TcpCommunicationSpiParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TcpCommunicationSpiParityTest.cs new file mode 100644 index 0000000000000..be8bd11af6487 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TcpCommunicationSpiParityTest.cs @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.ApiParity +{ + using System.Collections.Generic; + using Apache.Ignite.Core.Cache.Configuration; + using Apache.Ignite.Core.Communication.Tcp; + using NUnit.Framework; + + /// + /// Tests that .NET has all properties from Java configuration APIs. + /// + public class TcpCommunicationSpiParityTest + { + /** Known property name mappings. */ + private static readonly Dictionary KnownMappings = new Dictionary() + { + {"SocketReceiveBuffer", "SocketReceiveBufferSize"}, + {"SocketSendBuffer", "SocketSendBufferSize"} + }; + + /** Properties that are not needed on .NET side. */ + private static readonly string[] UnneededProperties = + { + // Java-specific. + "AddressResolver", + "Listener", + "run", + "ReceivedMessagesByType", + "ReceivedMessagesByNode", + "SentMessagesByType", + "SentMessagesByNode", + "SentMessagesCount", + "SentBytesCount", + "ReceivedMessagesCount", + "ReceivedBytesCount", + "OutboundMessagesQueueSize", + "resetMetrics", + "dumpStats", + "boundPort", + "SpiContext", + "simulateNodeFailure", + "cancel", + "order", + "onTimeout", + "endTime", + "id", + "connectionIndex", + "NodeFilter" + }; + + /** Properties that are missing on .NET side. */ + private static readonly string[] MissingProperties = {}; + + /// + /// Tests the cache configuration parity. + /// + [Test] + public void TestTcpCommunicationSpi() + { + ParityTest.CheckConfigurationParity( + @"modules\core\src\main\java\org\apache\ignite\spi\communication\tcp\TcpCommunicationSpi.java", + typeof(TcpCommunicationSpi), + UnneededProperties, + MissingProperties, + KnownMappings); + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index c26e5a346694e..f0f3b7cf956c5 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -186,9 +186,11 @@ public void TestAllConfigurationProperties() var com = (TcpCommunicationSpi) cfg.CommunicationSpi; var resCom = (TcpCommunicationSpi) resCfg.CommunicationSpi; Assert.AreEqual(com.AckSendThreshold, resCom.AckSendThreshold); + Assert.AreEqual(com.ConnectionsPerNode, resCom.ConnectionsPerNode); Assert.AreEqual(com.ConnectTimeout, resCom.ConnectTimeout); Assert.AreEqual(com.DirectBuffer, resCom.DirectBuffer); Assert.AreEqual(com.DirectSendBuffer, resCom.DirectSendBuffer); + Assert.AreEqual(com.FilterReachableAddresses, resCom.FilterReachableAddresses); Assert.AreEqual(com.IdleConnectionTimeout, resCom.IdleConnectionTimeout); Assert.AreEqual(com.LocalAddress, resCom.LocalAddress); Assert.AreEqual(com.LocalPort, resCom.LocalPort); @@ -197,12 +199,16 @@ public void TestAllConfigurationProperties() Assert.AreEqual(com.MessageQueueLimit, resCom.MessageQueueLimit); Assert.AreEqual(com.ReconnectCount, resCom.ReconnectCount); Assert.AreEqual(com.SelectorsCount, resCom.SelectorsCount); + Assert.AreEqual(com.SelectorSpins, resCom.SelectorSpins); + Assert.AreEqual(com.SharedMemoryPort, resCom.SharedMemoryPort); Assert.AreEqual(com.SlowClientQueueLimit, resCom.SlowClientQueueLimit); Assert.AreEqual(com.SocketReceiveBufferSize, resCom.SocketReceiveBufferSize); Assert.AreEqual(com.SocketSendBufferSize, resCom.SocketSendBufferSize); + Assert.AreEqual(com.SocketWriteTimeout, resCom.SocketWriteTimeout); Assert.AreEqual(com.TcpNoDelay, resCom.TcpNoDelay); Assert.AreEqual(com.UnacknowledgedMessagesBufferSize, resCom.UnacknowledgedMessagesBufferSize); - + Assert.AreEqual(com.UsePairedConnections, resCom.UsePairedConnections); + Assert.AreEqual(cfg.FailureDetectionTimeout, resCfg.FailureDetectionTimeout); Assert.AreEqual(cfg.SystemWorkerBlockedTimeout, resCfg.SystemWorkerBlockedTimeout); Assert.AreEqual(cfg.ClientFailureDetectionTimeout, resCfg.ClientFailureDetectionTimeout); @@ -750,7 +756,13 @@ private static IgniteConfiguration GetCustomConfig() TcpNoDelay = false, SlowClientQueueLimit = 98, SocketSendBufferSize = 2045, - UnacknowledgedMessagesBufferSize = 3450 + UnacknowledgedMessagesBufferSize = 3450, + ConnectionsPerNode = 12, + UsePairedConnections = true, + SharedMemoryPort = 1234, + SocketWriteTimeout = 2222, + SelectorSpins = 12, + FilterReachableAddresses = true }, FailureDetectionTimeout = TimeSpan.FromSeconds(3.5), SystemWorkerBlockedTimeout = TimeSpan.FromSeconds(8.5), diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Communication/Tcp/TcpCommunicationSpi.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Communication/Tcp/TcpCommunicationSpi.cs index d272906a39bf8..b070f9aadf804 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Communication/Tcp/TcpCommunicationSpi.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Communication/Tcp/TcpCommunicationSpi.cs @@ -40,6 +40,9 @@ public class TcpCommunicationSpi : ICommunicationSpi /// Default value of property. public const int DefaultAckSendThreshold = 16; + /// Default value of property. + public const int DefaultConnectionsPerNode = 1; + /// Default value of property. public static readonly TimeSpan DefaultConnectTimeout = TimeSpan.FromSeconds(5); @@ -49,6 +52,9 @@ public class TcpCommunicationSpi : ICommunicationSpi /// Default value of property. public const bool DefaultDirectSendBuffer = false; + /// Default value of property. + public const bool DefaultFilterReachableAddresses = false; + /// Default value of property. public static readonly TimeSpan DefaultIdleConnectionTimeout = TimeSpan.FromSeconds(30); @@ -70,31 +76,49 @@ public class TcpCommunicationSpi : ICommunicationSpi /// Default value of property. public static readonly int DefaultSelectorsCount = Math.Min(4, Environment.ProcessorCount); + /// Default value of property. + public const long DefaultSelectorSpins = 0; + + /// Default value of property. + public const int DefaultSharedMemoryPort = -1; + /// Default socket buffer size. public const int DefaultSocketBufferSize = 32 * 1024; + /// Default value of property. + public const long DefaultSocketWriteTimeout = 2000; + /// Default value of property. public const bool DefaultTcpNoDelay = true; + /// Default value of property. + public const bool DefaultUsePairedConnections = false; + /// /// Initializes a new instance of the class. /// public TcpCommunicationSpi() { AckSendThreshold = DefaultAckSendThreshold; + ConnectionsPerNode = DefaultConnectionsPerNode; ConnectTimeout = DefaultConnectTimeout; DirectBuffer = DefaultDirectBuffer; DirectSendBuffer = DefaultDirectSendBuffer; + FilterReachableAddresses = DefaultFilterReachableAddresses; IdleConnectionTimeout = DefaultIdleConnectionTimeout; LocalPort = DefaultLocalPort; LocalPortRange = DefaultLocalPortRange; MaxConnectTimeout = DefaultMaxConnectTimeout; MessageQueueLimit = DefaultMessageQueueLimit; ReconnectCount = DefaultReconnectCount; + SharedMemoryPort = DefaultSharedMemoryPort; SelectorsCount = DefaultSelectorsCount; + SelectorSpins = DefaultSelectorSpins; SocketReceiveBufferSize = DefaultSocketBufferSize; SocketSendBufferSize = DefaultSocketBufferSize; + SocketWriteTimeout = DefaultSocketWriteTimeout; TcpNoDelay = DefaultTcpNoDelay; + UsePairedConnections = DefaultUsePairedConnections; } /// @@ -104,9 +128,11 @@ public TcpCommunicationSpi() internal TcpCommunicationSpi(IBinaryRawReader reader) { AckSendThreshold = reader.ReadInt(); + ConnectionsPerNode = reader.ReadInt(); ConnectTimeout = reader.ReadLongAsTimespan(); DirectBuffer = reader.ReadBoolean(); DirectSendBuffer = reader.ReadBoolean(); + FilterReachableAddresses = reader.ReadBoolean(); IdleConnectionTimeout = reader.ReadLongAsTimespan(); LocalAddress = reader.ReadString(); LocalPort = reader.ReadInt(); @@ -115,15 +141,19 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) MessageQueueLimit = reader.ReadInt(); ReconnectCount = reader.ReadInt(); SelectorsCount = reader.ReadInt(); + SelectorSpins = reader.ReadLong(); + SharedMemoryPort = reader.ReadInt(); SlowClientQueueLimit = reader.ReadInt(); SocketReceiveBufferSize = reader.ReadInt(); SocketSendBufferSize = reader.ReadInt(); + SocketWriteTimeout = reader.ReadLong(); TcpNoDelay = reader.ReadBoolean(); UnacknowledgedMessagesBufferSize = reader.ReadInt(); + UsePairedConnections = reader.ReadBoolean(); } /// - /// Gets or sets the number of received messages per connection to node + /// Gets or sets the number of received messages per connection to node /// after which acknowledgment message is sent. /// [DefaultValue(DefaultAckSendThreshold)] @@ -136,14 +166,14 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) public TimeSpan ConnectTimeout { get; set; } /// - /// Gets or sets a value indicating whether to allocate direct (ByteBuffer.allocateDirect) + /// Gets or sets a value indicating whether to allocate direct (ByteBuffer.allocateDirect) /// or heap (ByteBuffer.allocate) buffer. /// [DefaultValue(DefaultDirectBuffer)] public bool DirectBuffer { get; set; } /// - /// Gets or sets a value indicating whether to allocate direct (ByteBuffer.allocateDirect) + /// Gets or sets a value indicating whether to allocate direct (ByteBuffer.allocateDirect) /// or heap (ByteBuffer.allocate) send buffer. /// [DefaultValue(DefaultDirectSendBuffer)] @@ -156,7 +186,7 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) public TimeSpan IdleConnectionTimeout { get; set; } /// - /// Gets or sets the local host address for socket binding. Note that one node could have + /// Gets or sets the local host address for socket binding. Note that one node could have /// additional addresses beside the loopback one. This configuration parameter is optional. /// public string LocalAddress { get; set; } @@ -193,7 +223,7 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) /// /// Gets or sets the message queue limit for incoming and outgoing messages. /// - /// When set to positive number send queue is limited to the configured value. + /// When set to positive number send queue is limited to the configured value. /// 0 disables the limitation. /// [DefaultValue(DefaultMessageQueueLimit)] @@ -216,11 +246,11 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) /// /// Gets or sets slow client queue limit. /// - /// When set to a positive number, communication SPI will monitor clients outbound message queue sizes + /// When set to a positive number, communication SPI will monitor clients outbound message queue sizes /// and will drop those clients whose queue exceeded this limit. /// /// Usually this value should be set to the same value as which controls - /// message back-pressure for server nodes. The default value for this parameter is 0 + /// message back-pressure for server nodes. The default value for this parameter is 0 /// which means unlimited. /// public int SlowClientQueueLimit { get; set; } @@ -230,7 +260,7 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) /// [DefaultValue(DefaultSocketBufferSize)] public int SocketReceiveBufferSize { get; set; } - + /// /// Gets or sets the size of the socket send buffer. /// @@ -250,21 +280,66 @@ internal TcpCommunicationSpi(IBinaryRawReader reader) public bool TcpNoDelay { get; set; } /// - /// Gets or sets the maximum number of stored unacknowledged messages per connection to node. - /// If number of unacknowledged messages exceeds this number + /// Gets or sets the maximum number of stored unacknowledged messages per connection to node. + /// If number of unacknowledged messages exceeds this number /// then connection to node is closed and reconnect is attempted. /// public int UnacknowledgedMessagesBufferSize { get; set; } + /// + /// Gets or sets the number of connections per node. + /// + [DefaultValue(DefaultConnectionsPerNode)] + public int ConnectionsPerNode { get; set; } + + /// + /// Gets or sets a value indicating whether separate connections should be used for incoming and outgoing data. + /// Set this to true if should maintain connection for outgoing + /// and incoming messages separately. In this case total number of connections between local and each remote + /// node is equals to * 2. + /// + public bool UsePairedConnections { get; set; } + + /// + /// Gets or sets a local port to accept shared memory connections. + /// + [DefaultValue(DefaultSharedMemoryPort)] + public int SharedMemoryPort { get; set; } + + /// + /// Gets or sets socket write timeout for TCP connection. If message can not be written to + /// socket within this time then connection is closed and reconnect is attempted. + /// + /// Default value is . + /// + [DefaultValue(DefaultSocketWriteTimeout)] + public long SocketWriteTimeout { get; set; } + + /// + /// Gets or sets a values that defines how many non-blocking selectors should be made. + /// Can be set to so selector threads will never block. + /// + /// Default value is . + /// + public long SelectorSpins { get; set; } + + /// + /// Gets or sets a value indicating whether filter for reachable addresses + /// should be enabled on creating tcp client. + /// + public bool FilterReachableAddresses { get; set; } + /// /// Writes this instance to the specified writer. /// internal void Write(IBinaryRawWriter writer) { writer.WriteInt(AckSendThreshold); + writer.WriteInt(ConnectionsPerNode); writer.WriteLong((long) ConnectTimeout.TotalMilliseconds); writer.WriteBoolean(DirectBuffer); writer.WriteBoolean(DirectSendBuffer); + writer.WriteBoolean(FilterReachableAddresses); writer.WriteLong((long) IdleConnectionTimeout.TotalMilliseconds); writer.WriteString(LocalAddress); writer.WriteInt(LocalPort); @@ -273,11 +348,15 @@ internal void Write(IBinaryRawWriter writer) writer.WriteInt(MessageQueueLimit); writer.WriteInt(ReconnectCount); writer.WriteInt(SelectorsCount); + writer.WriteLong(SelectorSpins); + writer.WriteInt(SharedMemoryPort); writer.WriteInt(SlowClientQueueLimit); writer.WriteInt(SocketReceiveBufferSize); writer.WriteInt(SocketSendBufferSize); + writer.WriteLong(SocketWriteTimeout); writer.WriteBoolean(TcpNoDelay); writer.WriteInt(UnacknowledgedMessagesBufferSize); + writer.WriteBoolean(UsePairedConnections); } } } From 8e819430af076b3b7dc5ac48274081cf3805dc37 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Mon, 19 Nov 2018 20:05:37 +0300 Subject: [PATCH 106/403] IGNITE-9501 Use correct protocol version --- .../distributed/dht/preloader/latch/ExchangeLatchManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java index 35c04fb7d7902..0308ff4198d95 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java @@ -64,7 +64,7 @@ public class ExchangeLatchManager { * Exchange latch V2 protocol introduces following optimization: * Joining nodes are explicitly excluded from possible latch participants. */ - public static final IgniteProductVersion PROTOCOL_V2_VERSION_SINCE = IgniteProductVersion.fromString("2.7.0"); + public static final IgniteProductVersion PROTOCOL_V2_VERSION_SINCE = IgniteProductVersion.fromString("2.5.3"); /** Logger. */ private final IgniteLogger log; From ba60d586bdca9f2c8ac1b1045e8e64a0ee954e33 Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Tue, 20 Nov 2018 15:15:16 +0300 Subject: [PATCH 107/403] IGNITE-10341 Added loss policy tests with persistence - Fixes #5444. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit eb8888561561f92343e0620082b81c757153b4be) Signed-off-by: Dmitriy Govorukhin --- ...gniteCachePartitionLossPolicySelfTest.java | 592 +++++++++++++----- 1 file changed, 421 insertions(+), 171 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java index f02563d0d47d3..cfe578db454a1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java @@ -19,10 +19,11 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; @@ -36,6 +37,8 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.CacheRebalancingEvent; import org.apache.ignite.events.Event; @@ -43,22 +46,25 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.TestDelayingCommunicationSpi; import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.P1; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; -import static org.apache.ignite.internal.processors.cache.GridCacheUtils.cacheId; /** * @@ -77,12 +83,14 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe private int backups; /** */ - private final AtomicBoolean delayPartExchange = new AtomicBoolean(); + private final AtomicBoolean delayPartExchange = new AtomicBoolean(false); /** */ private final TopologyChanger killSingleNode = new TopologyChanger( - false, asList(3), asList(0, 1, 2, 4), 0 - ); + false, singletonList(3), asList(0, 1, 2, 4), 0); + + /** */ + private boolean isPersistenceEnabled; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -101,9 +109,16 @@ false, asList(3), asList(0, 1, 2, 4), 0 cfg.setClientMode(client); + cfg.setCacheConfiguration(cacheConfiguration()); + cfg.setConsistentId(gridName); - cfg.setCacheConfiguration(cacheConfiguration()); + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(isPersistenceEnabled) + )); return cfg; } @@ -123,20 +138,26 @@ protected CacheConfiguration cacheConfiguration() { return cacheCfg; } - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopAllGrids(); - } - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); - cleanPersistenceDir(); - delayPartExchange.set(false); + partLossPlc = PartitionLossPolicy.IGNORE; + backups = 0; + + isPersistenceEnabled = false; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); } /** @@ -148,6 +169,17 @@ public void testReadOnlySafe() throws Exception { checkLostPartition(false, true, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadOnlySafeWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_ONLY_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(false, true, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -157,6 +189,19 @@ public void testReadOnlyAll() throws Exception { checkLostPartition(false, false, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadOnlyAllWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.READ_ONLY_ALL; + + isPersistenceEnabled = true; + + checkLostPartition(false, false, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -166,6 +211,17 @@ public void testReadWriteSafe() throws Exception { checkLostPartition(true, true, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -175,6 +231,19 @@ public void testReadWriteAll() throws Exception { checkLostPartition(true, false, killSingleNode); } + /** + * @throws Exception if failed. + */ + public void testReadWriteAllWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.READ_WRITE_ALL; + + isPersistenceEnabled = true; + + checkLostPartition(true, false, killSingleNode); + } + /** * @throws Exception if failed. */ @@ -184,6 +253,17 @@ public void testReadWriteSafeAfterKillTwoNodes() throws Exception { checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeAfterKillTwoNodesWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -193,6 +273,17 @@ public void testReadWriteSafeAfterKillTwoNodesWithDelay() throws Exception { checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 20)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeAfterKillTwoNodesWithDelayWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(false, asList(3, 2), asList(0, 1, 4), 20)); + } + /** * @throws Exception if failed. */ @@ -204,6 +295,21 @@ public void testReadWriteSafeWithBackupsAfterKillThreeNodes() throws Exception { checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2, 1), asList(0, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeWithBackupsAfterKillThreeNodesWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10043"); + + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + backups = 1; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2, 1), asList(0, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -213,6 +319,17 @@ public void testReadWriteSafeAfterKillCrd() throws Exception { checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeAfterKillCrdWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -224,6 +341,19 @@ public void testReadWriteSafeWithBackups() throws Exception { checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2), asList(0, 1, 4), 0)); } + /** + * @throws Exception if failed. + */ + public void testReadWriteSafeWithBackupsWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + backups = 1; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 2), asList(0, 1, 4), 0)); + } + /** * @throws Exception if failed. */ @@ -236,12 +366,81 @@ public void testReadWriteSafeWithBackupsAfterKillCrd() throws Exception { } /** - * @param topChanger topology changer. * @throws Exception if failed. */ - public void testIgnore(TopologyChanger topChanger) throws Exception { + public void testReadWriteSafeWithBackupsAfterKillCrdWithPersistence() throws Exception { + partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; + + backups = 1; + + isPersistenceEnabled = true; + + checkLostPartition(true, true, new TopologyChanger(true, asList(3, 0), asList(1, 2, 4), 0)); + } + + /** + * @throws Exception if failed. + */ + public void testIgnore() throws Exception { fail("https://issues.apache.org/jira/browse/IGNITE-5078"); + partLossPlc = PartitionLossPolicy.IGNORE; + + checkIgnore(killSingleNode); + } + + /** + * @throws Exception if failed. + */ + public void testIgnoreWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5078"); + + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.IGNORE; + + isPersistenceEnabled = true; + + checkIgnore(killSingleNode); + } + + /** + * @throws Exception if failed. + */ + public void testIgnoreKillThreeNodes() throws Exception { + partLossPlc = PartitionLossPolicy.IGNORE; + + // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078. + // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed. + // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0); + TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, asList(1, 2, 3), singletonList(0), 0); + + checkIgnore(onlyCrdIsAlive); + } + + /** + * @throws Exception if failed. + */ + public void testIgnoreKillThreeNodesWithPersistence() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10041"); + + partLossPlc = PartitionLossPolicy.IGNORE; + + isPersistenceEnabled = true; + + // TODO aliveNodes should include node 4, but it fails due to https://issues.apache.org/jira/browse/IGNITE-5078. + // TODO need to add 4 to the aliveNodes after IGNITE-5078 is fixed. + // TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, Arrays.asList(1, 2, 3), Arrays.asList(0, 4), 0); + TopologyChanger onlyCrdIsAlive = new TopologyChanger(false, asList(1, 2, 3), singletonList(0), 0); + + checkIgnore(onlyCrdIsAlive); + } + + /** + * @param topChanger topology changer. + * @throws Exception if failed. + */ + private void checkIgnore(TopologyChanger topChanger) throws Exception { topChanger.changeTopology(); for (Ignite ig : G.allGrids()) { @@ -270,42 +469,80 @@ public void testIgnore(TopologyChanger topChanger) throws Exception { private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger topChanger) throws Exception { assert partLossPlc != null; - int part = topChanger.changeTopology().get(0); + List lostParts = topChanger.changeTopology(); + + // Wait for all grids (servers and client) have same topology version + // to make sure that all nodes received map with lost partition. + boolean success = GridTestUtils.waitForCondition(() -> { + AffinityTopologyVersion last = null; + for (Ignite ig : G.allGrids()) { + AffinityTopologyVersion ver = ((IgniteEx)ig).context().cache().context().exchange().readyAffinityVersion(); + + if (last != null && !last.equals(ver)) + return false; + + last = ver; + } + + return true; + }, 10000); + + assertTrue("Failed to wait for new topology", success); for (Ignite ig : G.allGrids()) { info("Checking node: " + ig.cluster().localNode().id()); IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); - verifyCacheOps(canWrite, safe, part, ig); + verifyLostPartitions(ig, lostParts); - // Check we can read and write to lost partition in recovery mode. - IgniteCache recoverCache = cache.withPartitionRecover(); + verifyCacheOps(canWrite, safe, ig); - for (int lostPart : recoverCache.lostPartitions()) { - recoverCache.get(lostPart); - recoverCache.put(lostPart, lostPart); - } + validateQuery(safe, ig); - // Check that writing in recover mode does not clear partition state. - verifyCacheOps(canWrite, safe, part, ig); + // TODO withPartitionRecover doesn't work with BLT - https://issues.apache.org/jira/browse/IGNITE-10041. + if (!isPersistenceEnabled) { + // Check we can read and write to lost partition in recovery mode. + IgniteCache recoverCache = cache.withPartitionRecover(); - // Validate queries. - validateQuery(safe, ig); + for (int lostPart : recoverCache.lostPartitions()) { + recoverCache.get(lostPart); + recoverCache.put(lostPart, lostPart); + } + + // Check that writing in recover mode does not clear partition state. + verifyLostPartitions(ig, lostParts); + + verifyCacheOps(canWrite, safe, ig); + + validateQuery(safe, ig); + } } - checkNewNode(true, canWrite, safe, part); - checkNewNode(false, canWrite, safe, part); + checkNewNode(true, canWrite, safe); + checkNewNode(false, canWrite, safe); + + // Bring all nodes back. + for (int i : topChanger.killNodes) { + IgniteEx grd = startGrid(i); - // Check that partition state does not change after we start a new node. - IgniteEx grd = startGrid(3); + info("Newly started node: " + grd.cluster().localNode().id()); - info("Newly started node: " + grd.cluster().localNode().id()); + // Check that partition state does not change after we start each node. + // TODO With persistence enabled LOST partitions become OWNING after a node joins back - https://issues.apache.org/jira/browse/IGNITE-10044. + if (!isPersistenceEnabled) { + for (Ignite ig : G.allGrids()) { + verifyCacheOps(canWrite, safe, ig); - for (Ignite ig : G.allGrids()) - verifyCacheOps(canWrite, safe, part, ig); + // TODO Query effectively waits for rebalance due to https://issues.apache.org/jira/browse/IGNITE-10057 + // TODO and after resetLostPartition there is another OWNING copy in the cluster due to https://issues.apache.org/jira/browse/IGNITE-10058. + // TODO Uncomment after https://issues.apache.org/jira/browse/IGNITE-10058 is fixed. +// validateQuery(safe, ig); + } + } + } - ignite(4).resetLostPartitions(Collections.singletonList(DEFAULT_CACHE_NAME)); + ignite(4).resetLostPartitions(singletonList(DEFAULT_CACHE_NAME)); awaitPartitionMapExchange(true, true, null); @@ -321,6 +558,16 @@ private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger cache.put(i, i); } + + for (int i = 0; i < parts; i++) { + checkQueryPasses(ig, false, i); + + if (shouldExecuteLocalQuery(ig, i)) + checkQueryPasses(ig, true, i); + + } + + checkQueryPasses(ig, false); } } @@ -328,25 +575,23 @@ private void checkLostPartition(boolean canWrite, boolean safe, TopologyChanger * @param client Client flag. * @param canWrite Can write flag. * @param safe Safe flag. - * @param part List of lost partitions. * @throws Exception If failed to start a new node. */ private void checkNewNode( boolean client, boolean canWrite, - boolean safe, - int part + boolean safe ) throws Exception { this.client = client; try { - IgniteEx cl = startGrid("newNode"); + IgniteEx cl = (IgniteEx)startGrid("newNode"); - CacheGroupContext grpCtx = cl.context().cache().cacheGroup(cacheId(DEFAULT_CACHE_NAME)); + CacheGroupContext grpCtx = cl.context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME)); assertTrue(grpCtx.needsRecovery()); - verifyCacheOps(canWrite, safe, part, cl); + verifyCacheOps(canWrite, safe, cl); validateQuery(safe, cl); } @@ -358,20 +603,26 @@ private void checkNewNode( } /** - * + * @param node Node. + * @param lostParts Lost partition IDs. + */ + private void verifyLostPartitions(Ignite node, List lostParts) { + IgniteCache cache = node.cache(DEFAULT_CACHE_NAME); + + Set actualSortedLostParts = new TreeSet<>(cache.lostPartitions()); + Set expSortedLostParts = new TreeSet<>(lostParts); + + assertEqualsCollections(expSortedLostParts, actualSortedLostParts); + } + + /** * @param canWrite {@code True} if writes are allowed. * @param safe {@code True} if lost partition should trigger exception. - * @param part Lost partition ID. * @param ig Ignite instance. */ - private void verifyCacheOps(boolean canWrite, boolean safe, int part, Ignite ig) { + private void verifyCacheOps(boolean canWrite, boolean safe, Ignite ig) { IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); - Collection lost = cache.lostPartitions(); - - assertTrue("Failed to find expected lost partition [exp=" + part + ", lost=" + lost + ']', - lost.contains(part)); - int parts = ig.affinity(DEFAULT_CACHE_NAME).partitions(); // Check read. @@ -419,7 +670,7 @@ private void verifyCacheOps(boolean canWrite, boolean safe, int part, Ignite ig) * @param nodes List of nodes to find partition. * @return List of partitions that aren't primary or backup for specified nodes. */ - protected List noPrimaryOrBackupPartition(List nodes) { + private List noPrimaryOrBackupPartition(List nodes) { Affinity aff = ignite(4).affinity(DEFAULT_CACHE_NAME); List parts = new ArrayList<>(); @@ -444,127 +695,6 @@ protected List noPrimaryOrBackupPartition(List nodes) { return parts; } - /** */ - private class TopologyChanger { - /** Flag to delay partition exchange */ - private boolean delayExchange; - - /** List of nodes to kill */ - private List killNodes; - - /** List of nodes to be alive */ - private List aliveNodes; - - /** Delay between node stops */ - private long stopDelay; - - /** - * @param delayExchange Flag for delay partition exchange. - * @param killNodes List of nodes to kill. - * @param aliveNodes List of nodes to be alive. - * @param stopDelay Delay between stopping nodes. - */ - public TopologyChanger( - boolean delayExchange, - List killNodes, - List aliveNodes, - long stopDelay - ) { - this.delayExchange = delayExchange; - this.killNodes = killNodes; - this.aliveNodes = aliveNodes; - this.stopDelay = stopDelay; - } - - /** - * @return Lost partition ID. - * @throws Exception If failed. - */ - protected List changeTopology() throws Exception { - startGrids(4); - - Affinity aff = ignite(0).affinity(DEFAULT_CACHE_NAME); - - for (int i = 0; i < aff.partitions(); i++) - ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i); - - client = true; - - startGrid(4); - - client = false; - - for (int i = 0; i < 5; i++) - info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']'); - - awaitPartitionMapExchange(); - - final List parts = noPrimaryOrBackupPartition(aliveNodes); - - if (parts.isEmpty()) - throw new IllegalStateException("No partition on nodes: " + killNodes); - - final List> lostMap = new ArrayList<>(); - - for (int i : aliveNodes) { - HashMap semaphoreMap = new HashMap<>(); - - for (Integer part : parts) - semaphoreMap.put(part, new Semaphore(0)); - - lostMap.add(semaphoreMap); - - grid(i).events().localListen(new P1() { - @Override public boolean apply(Event evt) { - assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; - - CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt; - - if (F.eq(DEFAULT_CACHE_NAME, cacheEvt.cacheName())) { - if (semaphoreMap.containsKey(cacheEvt.partition())) - semaphoreMap.get(cacheEvt.partition()).release(); - } - - return true; - } - }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST); - } - - if (delayExchange) - delayPartExchange.set(true); - - ExecutorService executor = Executors.newFixedThreadPool(killNodes.size()); - - for (Integer node : killNodes) { - executor.submit(new Runnable() { - @Override public void run() { - grid(node).close(); - } - }); - - Thread.sleep(stopDelay); - } - - executor.shutdown(); - - delayPartExchange.set(false); - - Thread.sleep(5_000L); - - for (Map map : lostMap) { - for (Map.Entry entry : map.entrySet()) - assertTrue("Failed to wait for partition LOST event for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); - } - - for (Map map : lostMap) { - for (Map.Entry entry : map.entrySet()) - assertFalse("Partition LOST event raised twice for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); - } - - return parts; - } - } - /** * Validate query execution on a node. * @@ -685,4 +815,124 @@ protected void checkQueryFails(Ignite node, boolean loc, int... parts) { // TODO Need to add an actual check after https://issues.apache.org/jira/browse/IGNITE-9902 is fixed. // No-op. } + + /** */ + private class TopologyChanger { + /** Flag to delay partition exchange */ + private boolean delayExchange; + + /** List of nodes to kill */ + private List killNodes; + + /** List of nodes to be alive */ + private List aliveNodes; + + /** Delay between node stops */ + private long stopDelay; + + /** + * @param delayExchange Flag for delay partition exchange. + * @param killNodes List of nodes to kill. + * @param aliveNodes List of nodes to be alive. + * @param stopDelay Delay between stopping nodes. + */ + private TopologyChanger(boolean delayExchange, List killNodes, List aliveNodes, + long stopDelay) { + this.delayExchange = delayExchange; + this.killNodes = killNodes; + this.aliveNodes = aliveNodes; + this.stopDelay = stopDelay; + } + + /** + * @return Lost partition ID. + * @throws Exception If failed. + */ + private List changeTopology() throws Exception { + startGrids(4); + + if (isPersistenceEnabled) + grid(0).cluster().active(true); + + Affinity aff = ignite(0).affinity(DEFAULT_CACHE_NAME); + + for (int i = 0; i < aff.partitions(); i++) + ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i); + + client = true; + + startGrid(4); + + client = false; + + for (int i = 0; i < 5; i++) + info(">>> Node [idx=" + i + ", nodeId=" + ignite(i).cluster().localNode().id() + ']'); + + awaitPartitionMapExchange(); + + final List parts = noPrimaryOrBackupPartition(aliveNodes); + + if (parts.isEmpty()) + throw new IllegalStateException("No partition on nodes: " + killNodes); + + final List> lostMap = new ArrayList<>(); + + for (int i : aliveNodes) { + HashMap semaphoreMap = new HashMap<>(); + + for (Integer part : parts) + semaphoreMap.put(part, new Semaphore(0)); + + lostMap.add(semaphoreMap); + + grid(i).events().localListen(new P1() { + @Override public boolean apply(Event evt) { + assert evt.type() == EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST; + + CacheRebalancingEvent cacheEvt = (CacheRebalancingEvent)evt; + + if (F.eq(DEFAULT_CACHE_NAME, cacheEvt.cacheName())) { + if (semaphoreMap.containsKey(cacheEvt.partition())) + semaphoreMap.get(cacheEvt.partition()).release(); + } + + return true; + } + }, EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST); + } + + if (delayExchange) + delayPartExchange.set(true); + + ExecutorService executor = Executors.newFixedThreadPool(killNodes.size()); + + for (Integer node : killNodes) { + executor.submit(new Runnable() { + @Override public void run() { + grid(node).close(); + } + }); + + Thread.sleep(stopDelay); + } + + executor.shutdown(); + + delayPartExchange.set(false); + + Thread.sleep(5_000L); + + for (Map map : lostMap) { + for (Map.Entry entry : map.entrySet()) + assertTrue("Failed to wait for partition LOST event for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); + } + + for (Map map : lostMap) { + for (Map.Entry entry : map.entrySet()) + assertFalse("Partition LOST event raised twice for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); + } + + return parts; + } + } } From 93030db4eec02031819247b86e939bf70f932cb2 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 20 Nov 2018 15:45:39 +0300 Subject: [PATCH 108/403] IGNITE-9999 Added verbose logging for node recovery - Fixes #5371 (cherry picked from commit b48a291) --- .../apache/ignite/IgniteSystemProperties.java | 7 + .../processors/cache/CacheGroupContext.java | 148 ------- .../processors/cache/GridCacheProcessor.java | 44 +- .../cache/IgniteCacheOffheapManager.java | 11 + .../cache/IgniteCacheOffheapManagerImpl.java | 7 + .../GridDhtPartitionsExchangeFuture.java | 21 +- .../DatabaseLifecycleListener.java | 43 +- .../GridCacheDatabaseSharedManager.java | 391 ++++++++---------- .../persistence/GridCacheOffheapManager.java | 140 +++++++ .../IgniteCacheDatabaseSharedManager.java | 12 +- .../cache/persistence/wal/FileWALPointer.java | 2 +- .../serializer/RecordDataV1Serializer.java | 2 +- .../db/IgniteLogicalRecoveryTest.java | 284 +++++++------ .../IgnitePdsDiskErrorsRecoveringTest.java | 58 --- ...IgniteNodeStoppedDuringDisableWALTest.java | 3 +- .../persistence/db/wal/WalCompactionTest.java | 13 +- 16 files changed, 603 insertions(+), 583 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 692cfe489a8d9..943583bcf1d72 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1031,6 +1031,13 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_ALLOW_START_CACHES_IN_PARALLEL = "IGNITE_ALLOW_START_CACHES_IN_PARALLEL"; + /** + * Allows to log additional information about all restored partitions after binary and logical recovery phases. + * + * Default is {@code true}. + */ + public static final String IGNITE_RECOVERY_VERBOSE_LOGGING = "IGNITE_RECOVERY_VERBOSE_LOGGING"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 93ba69c25960d..ea594bc7dc6e1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -22,7 +22,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -42,17 +41,11 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; -import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; -import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; -import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.query.continuous.CounterSkipContext; import org.apache.ignite.internal.processors.query.QueryUtils; @@ -171,9 +164,6 @@ public class CacheGroupContext { /** Flag indicates that cache group is under recovering and not attached to topology. */ private final AtomicBoolean recoveryMode; - /** Flag indicates that all group partitions have restored their state from page memory / disk. */ - private volatile boolean partitionStatesRestored; - /** * @param ctx Context. * @param grpId Group ID. @@ -790,144 +780,6 @@ public void finishRecovery( } } - /** - * Pre-create partitions that resides in page memory or WAL and restores their state. - */ - public long restorePartitionStates(Map partitionRecoveryStates) throws IgniteCheckedException { - if (isLocal() || !affinityNode() || !dataRegion().config().isPersistenceEnabled()) - return 0; - - if (partitionStatesRestored) - return 0; - - long processed = 0; - - PageMemoryEx pageMem = (PageMemoryEx)dataRegion().pageMemory(); - - for (int p = 0; p < affinity().partitions(); p++) { - PartitionRecoverState recoverState = partitionRecoveryStates.get(new GroupPartitionId(grpId, p)); - - if (ctx.pageStore().exists(grpId, p)) { - ctx.pageStore().ensure(grpId, p); - - if (ctx.pageStore().pages(grpId, p) <= 1) { - if (log.isDebugEnabled()) - log.debug("Skipping partition on recovery (pages less than 1) " + - "[grp=" + cacheOrGroupName() + ", p=" + p + "]"); - - continue; - } - - if (log.isDebugEnabled()) - log.debug("Creating partition on recovery (exists in page store) " + - "[grp=" + cacheOrGroupName() + ", p=" + p + "]"); - - processed++; - - GridDhtLocalPartition part = topology().forceCreatePartition(p); - - offheap().onPartitionInitialCounterUpdated(p, 0); - - ctx.database().checkpointReadLock(); - - try { - long partMetaId = pageMem.partitionMetaPageId(grpId, p); - long partMetaPage = pageMem.acquirePage(grpId, partMetaId); - - try { - long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage); - - boolean changed = false; - - try { - PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr); - - if (recoverState != null) { - io.setPartitionState(pageAddr, (byte) recoverState.stateId()); - - changed = updateState(part, recoverState.stateId()); - - if (recoverState.stateId() == GridDhtPartitionState.OWNING.ordinal() - || (recoverState.stateId() == GridDhtPartitionState.MOVING.ordinal() - && part.initialUpdateCounter() < recoverState.updateCounter())) { - part.initialUpdateCounter(recoverState.updateCounter()); - - changed = true; - } - - if (log.isInfoEnabled()) - log.warning("Restored partition state (from WAL) " + - "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + - ", updCntr=" + part.initialUpdateCounter() + "]"); - } - else { - int stateId = (int) io.getPartitionState(pageAddr); - - changed = updateState(part, stateId); - - if (log.isDebugEnabled()) - log.debug("Restored partition state (from page memory) " + - "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + - ", updCntr=" + part.initialUpdateCounter() + ", stateId=" + stateId + "]"); - } - } - finally { - pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed); - } - } - finally { - pageMem.releasePage(grpId, partMetaId, partMetaPage); - } - } - finally { - ctx.database().checkpointReadUnlock(); - } - } - else if (recoverState != null) { - GridDhtLocalPartition part = topology().forceCreatePartition(p); - - offheap().onPartitionInitialCounterUpdated(p, recoverState.updateCounter()); - - updateState(part, recoverState.stateId()); - - processed++; - - if (log.isDebugEnabled()) - log.debug("Restored partition state (from WAL) " + - "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + - ", updCntr=" + part.initialUpdateCounter() + "]"); - } - else { - if (log.isDebugEnabled()) - log.debug("Skipping partition on recovery (no page store OR wal state) " + - "[grp=" + cacheOrGroupName() + ", p=" + p + "]"); - } - } - - partitionStatesRestored = true; - - return processed; - } - - /** - * @param part Partition to restore state for. - * @param stateId State enum ordinal. - * @return Updated flag. - */ - private boolean updateState(GridDhtLocalPartition part, int stateId) { - if (stateId != -1) { - GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId); - - assert state != null; - - part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state); - - return true; - } - - return false; - } - /** * @return {@code True} if current cache group is in recovery mode. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index ed08806ade6d4..ed021ba39408d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -108,6 +108,8 @@ import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage; +import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; @@ -5454,6 +5456,15 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da /** Set of QuerySchema's saved on recovery. It's needed if cache query schema has changed after node joined to topology.*/ private final Map querySchemas = new ConcurrentHashMap<>(); + /** {@inheritDoc} */ + @Override public void onBaselineChange() { + onKernalStopCaches(true); + + stopCaches(true); + + sharedCtx.database().cleanupRestoredCaches(); + } + /** {@inheritDoc} */ @Override public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException { restoreCacheConfigurations(); @@ -5466,7 +5477,7 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da } /** {@inheritDoc} */ - @Override public void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + @Override public void afterBinaryMemoryRestore(GridCacheDatabaseSharedManager.RestoreBinaryState binaryState) throws IgniteCheckedException { for (DynamicCacheDescriptor cacheDescriptor : persistentCaches()) { // Skip MVCC caches. // TODO: https://issues.apache.org/jira/browse/IGNITE-10052 @@ -5478,6 +5489,37 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da querySchemas.put(cacheDescriptor.cacheId(), cacheDescriptor.schema().copy()); } } + + /** {@inheritDoc} */ + @Override public void afterLogicalUpdatesApplied(GridCacheDatabaseSharedManager.RestoreLogicalState logicalState) throws IgniteCheckedException { + restorePartitionStates(cacheGroups(), logicalState.partitionRecoveryStates()); + } + + /** + * @param forGroups Cache groups. + * @param partitionStates Partition states. + * @throws IgniteCheckedException If failed. + */ + private void restorePartitionStates( + Collection forGroups, + Map partitionStates + ) throws IgniteCheckedException { + long startRestorePart = U.currentTimeMillis(); + + if (log.isInfoEnabled()) + log.info("Restoring partition state for local groups."); + + long totalProcessed = 0; + + for (CacheGroupContext grp : forGroups) + totalProcessed += grp.offheap().restorePartitionStates(partitionStates); + + if (log.isInfoEnabled()) + log.info("Finished restoring partition state for local groups [" + + "groupsProcessed=" + forGroups.size() + + ", partitionsProcessed=" + totalProcessed + + ", time=" + (U.currentTimeMillis() - startRestorePart) + "ms]"); + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index 58f05d2e062fa..3b6a2ca12ea81 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -31,6 +31,8 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow; import org.apache.ignite.internal.processors.cache.persistence.RootPage; import org.apache.ignite.internal.processors.cache.persistence.RowStore; +import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccUpdateResult; @@ -81,6 +83,15 @@ public interface IgniteCacheOffheapManager { */ public void stop(); + /** + * Pre-create partitions that resides in page memory or WAL and restores their state. + * + * @param partitionRecoveryStates Partition recovery states. + * @return Number of processed partitions. + * @throws IgniteCheckedException If failed. + */ + long restorePartitionStates(Map partitionRecoveryStates) throws IgniteCheckedException; + /** * Partition counter update callback. May be overridden by plugin-provided subclasses. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index c450d0265a437..5c3398cafb806 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -58,6 +58,8 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow; import org.apache.ignite.internal.processors.cache.persistence.RootPage; import org.apache.ignite.internal.processors.cache.persistence.RowStore; +import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO; @@ -256,6 +258,11 @@ protected void initDataStructures() throws IgniteCheckedException { } } + /** {@inheritDoc} */ + @Override public long restorePartitionStates(Map partitionRecoveryStates) throws IgniteCheckedException { + return 0; // No-op. + } + /** {@inheritDoc} */ @Override public void onKernalStop() { busyLock.block(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 4c080682aec31..64eadfcdb6cd9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -92,6 +92,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator; import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; +import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; @@ -882,23 +883,11 @@ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedExce cctx.exchange().exchangerBlockingSectionBegin(); try { - // Stop all recovered caches and groups. - cctx.cache().onKernalStopCaches(true); + List listeners = cctx.kernalContext().internalSubscriptionProcessor() + .getDatabaseListeners(); - cctx.cache().stopCaches(true); - - cctx.database().cleanupRestoredCaches(); - - for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) { - if (CU.isPersistentCache(desc.cacheConfiguration(), - cctx.gridConfig().getDataStorageConfiguration())) { - // Perform cache init from scratch. - cctx.cache().preparePageStore(desc, true); - } - } - - // Set initial node started marker. - cctx.database().nodeStart(null); + for (DatabaseLifecycleListener lsnr : listeners) + lsnr.onBaselineChange(); } finally { cctx.exchange().exchangerBlockingSectionEnd(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java index 1f7ba84a05d02..676210964e051 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java @@ -22,6 +22,7 @@ /** * */ +@SuppressWarnings("RedundantThrows") public interface DatabaseLifecycleListener { /** * Callback executed when data regions become to start-up. @@ -29,7 +30,15 @@ public interface DatabaseLifecycleListener { * @param mgr Database shared manager. * @throws IgniteCheckedException If failed. */ - default void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; + public default void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {} + + /** + * Callback executed when node detected that baseline topology is changed and node is not in that baseline. + * It's useful to cleanup and invalidate all available data restored at that moment. + * + * @throws IgniteCheckedException If failed. + */ + public default void onBaselineChange() throws IgniteCheckedException {} /** * Callback executed right before node become perform binary recovery. @@ -37,30 +46,46 @@ public interface DatabaseLifecycleListener { * @param mgr Database shared manager. * @throws IgniteCheckedException If failed. */ - default void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; + public default void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {} /** * Callback executed when binary memory has fully restored and WAL logging is resumed. * - * @param mgr Database shared manager. + * @param binaryState Result of binary recovery. * @throws IgniteCheckedException If failed. */ - default void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; + public default void afterBinaryMemoryRestore(GridCacheDatabaseSharedManager.RestoreBinaryState binaryState) + throws IgniteCheckedException {} /** + * Callback executed when all logical updates were applied and page memory become to fully consistent state. * - * @param mgr - * @throws IgniteCheckedException + * @param logicalState Result of logical recovery. + * @throws IgniteCheckedException If failed. */ - default void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; + public default void afterLogicalUpdatesApplied(GridCacheDatabaseSharedManager.RestoreLogicalState logicalState) + throws IgniteCheckedException {} /** + * Callback executed when all physical updates are applied and we are ready to write new physical records + * during logical recovery. + * * @param mgr Database shared manager. + * @throws IgniteCheckedException If failed. */ - default void afterInitialise(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {}; + public default void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {} /** + * Callback executed after all data regions are initialized. + * + * @param mgr Database shared manager. + */ + public default void afterInitialise(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {} + + /** + * Callback executed before shared manager will be stopped. + * * @param mgr Database shared manager. */ - default void beforeStop(IgniteCacheDatabaseSharedManager mgr) {}; + public default void beforeStop(IgniteCacheDatabaseSharedManager mgr) {} } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index afba6bdb4d61e..6e6bb0ec5ebd5 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -93,7 +93,6 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; -import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; @@ -130,6 +129,7 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.processors.port.GridPortRecord; @@ -150,6 +150,7 @@ import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteOutClosure; @@ -163,7 +164,6 @@ import static java.nio.file.StandardOpenOption.READ; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT; -import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD; import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; @@ -187,9 +187,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** Skip sync. */ private final boolean skipSync = IgniteSystemProperties.getBoolean(IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC); - /** */ - private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false); - /** */ private final int walRebalanceThreshold = IgniteSystemProperties.getInteger( IGNITE_PDS_WAL_REBALANCE_THRESHOLD, 500_000); @@ -214,9 +211,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** Checkpoint file name pattern. */ public static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin"); - /** Node started file suffix. */ - public static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin"; - /** */ private static final String MBEAN_NAME = "DataStorageMetrics"; @@ -358,6 +352,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** Timeout for checkpoint read lock acquisition in milliseconds. */ private volatile long checkpointReadLockTimeout; + /** Flag allows to log additional information about partitions during recovery phases. */ + private final boolean recoveryVerboseLogging = IgniteSystemProperties.getBoolean( + IgniteSystemProperties.IGNITE_RECOVERY_VERBOSE_LOGGING, true); + /** * @param ctx Kernal context. */ @@ -480,6 +478,8 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi final GridKernalContext kernalCtx = cctx.kernalContext(); if (!kernalCtx.clientNode()) { + kernalCtx.internalSubscriptionProcessor().registerDatabaseListener(new MetastorageRecoveryLifecycle()); + checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log); cpHistory = new CheckpointHistory(kernalCtx); @@ -539,7 +539,7 @@ private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfi continue; if (log.isInfoEnabled()) - log.info("Page memory " + region + " for " + grpDesc + " has invalidated."); + log.info("Page memory " + region.config().getName() + " for " + grpDesc + " has invalidated."); int partitions = grpDesc.config().getAffinity().partitions(); @@ -891,31 +891,40 @@ private MetaStorage createMetastorage(boolean readOnly) throws IgniteCheckedExce * @return Last seen WAL pointer during binary memory recovery. * @throws IgniteCheckedException If failed. */ - private WALPointer restoreBinaryMemory(Predicate cacheGroupsPredicate) throws IgniteCheckedException { - assert !cctx.kernalContext().clientNode(); - + private RestoreBinaryState restoreBinaryMemory(Predicate cacheGroupsPredicate) throws IgniteCheckedException { long time = System.currentTimeMillis(); - checkpointReadLock(); - try { + log.info("Starting binary memory restore for: " + cctx.cache().cacheGroupDescriptors().keySet()); + + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) + lsnr.beforeBinaryMemoryRestore(this); + CheckpointStatus status = readCheckpointStatus(); // First, bring memory to the last consistent checkpoint state if needed. // This method should return a pointer to the last valid record in the WAL. - WALPointer restored = performBinaryMemoryRestore(status, cacheGroupsPredicate, true); + RestoreBinaryState binaryState = performBinaryMemoryRestore(status, cacheGroupsPredicate, true); + + WALPointer restored = binaryState.lastReadRecordPointer(); if (restored == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { throw new StorageException("The memory cannot be restored. The critical part of WAL archive is missing " + "[tailWalPtr=" + restored + ", endPtr=" + status.endPtr + ']'); } + else if (restored != null) + U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restored + ']'); + + // Wal logging is now available. + cctx.wal().resumeLogging(restored); - nodeStart(restored); + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) + lsnr.afterBinaryMemoryRestore(binaryState); if (log.isInfoEnabled()) log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms."); - return restored; + return binaryState; } catch (IgniteCheckedException e) { if (X.hasCause(e, StorageException.class, IOException.class)) @@ -923,97 +932,6 @@ private WALPointer restoreBinaryMemory(Predicate cacheGroupsPredicate) throw e; } - finally { - checkpointReadUnlock(); - } - } - - /** {@inheritDoc} */ - @Override public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException { - FileWALPointer p = (FileWALPointer)(ptr == null ? CheckpointStatus.NULL_PTR : ptr); - - String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX; - String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX; - - ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE); - buf.order(ByteOrder.nativeOrder()); - - try { - try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), tmpFileName).toFile(), - StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { - buf.putLong(p.index()); - - buf.putInt(p.fileOffset()); - - buf.putInt(p.length()); - - buf.flip(); - - io.writeFully(buf); - - buf.clear(); - - io.force(true); - } - - Files.move(Paths.get(cpDir.getAbsolutePath(), tmpFileName), Paths.get(cpDir.getAbsolutePath(), fileName)); - } - catch (IOException e) { - throw new StorageException("Failed to write node start marker: " + ptr, e); - } - } - - /** - * Collects memory recovery pointers from node started files. See {@link #nodeStart(WALPointer)}. - * Each pointer associated with timestamp extracted from file. - * Tuples are sorted by timestamp. - * - * @return Sorted list of tuples (node started timestamp, memory recovery pointer). - * - * @throws IgniteCheckedException If failed. - */ - public List> nodeStartedPointers() throws IgniteCheckedException { - List> res = new ArrayList<>(); - - try (DirectoryStream nodeStartedFiles = Files.newDirectoryStream( - cpDir.toPath(), - path -> path.toFile().getName().endsWith(NODE_STARTED_FILE_NAME_SUFFIX)) - ) { - ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE); - buf.order(ByteOrder.nativeOrder()); - - for (Path path : nodeStartedFiles) { - File f = path.toFile(); - - String name = f.getName(); - - Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length())); - - try (FileIO io = ioFactory.create(f, READ)) { - io.readFully(buf); - - buf.flip(); - - FileWALPointer ptr = new FileWALPointer( - buf.getLong(), buf.getInt(), buf.getInt()); - - res.add(new T2<>(ts, ptr)); - - buf.clear(); - } - catch (IOException e) { - throw new StorageException("Failed to read node started marker file: " + f.getAbsolutePath(), e); - } - } - } - catch (IOException e) { - throw new StorageException("Failed to retreive node started files.", e); - } - - // Sort start markers by file timestamp. - res.sort(Comparator.comparingLong(IgniteBiTuple::get1)); - - return res; } /** {@inheritDoc} */ @@ -1353,7 +1271,7 @@ private void shutdownCheckpointer(boolean cancel) { cctx.database().checkpointReadLock(); try { - cacheGroup.restorePartitionStates(Collections.emptyMap()); + cacheGroup.offheap().restorePartitionStates(Collections.emptyMap()); if (cacheGroup.localStartVersion().equals(fut.initialVersion())) cacheGroup.topology().afterStateRestored(fut.initialVersion()); @@ -1492,14 +1410,12 @@ private void prepareIndexRebuildFuture(int cacheId) { for (IgniteBiTuple tup : stoppedGrps) { CacheGroupContext grp = tup.get1(); - if (grp.affinityNode()) { - try { - cctx.pageStore().shutdownForCacheGroup(grp, tup.get2()); - } - catch (IgniteCheckedException e) { - U.error(log, "Failed to gracefully clean page store resources for destroyed cache " + - "[cache=" + grp.cacheOrGroupName() + "]", e); - } + try { + cctx.pageStore().shutdownForCacheGroup(grp, tup.get2()); + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to gracefully clean page store resources for destroyed cache " + + "[cache=" + grp.cacheOrGroupName() + "]", e); } } } @@ -1967,32 +1883,16 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC // Preform early regions startup before restoring state. initAndStartRegions(kctx.config().getDataStorageConfiguration()); - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) - lsnr.beforeBinaryMemoryRestore(this); - - log.info("Starting binary memory restore for: " + cctx.cache().cacheGroupDescriptors().keySet()); - - cctx.pageStore().initializeForMetastorage(); - // Restore binary memory for all not WAL disabled cache groups. - WALPointer restored = restoreBinaryMemory( + restoreBinaryMemory( g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g) ); - if (restored != null) - U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restored + ']'); - - for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx)) - lsnr.afterBinaryMemoryRestore(this); - - cctx.wal().resumeLogging(restored); - - // We should log this record to ensure that node start marker pointer will be found in compacted segment. - cctx.wal().log(new MemoryRecoveryRecord(System.currentTimeMillis())); + if (recoveryVerboseLogging && log.isInfoEnabled()) { + log.info("Partition states information after BINARY RECOVERY phase:"); - assert metaStorage == null; - - metaStorage = createMetastorage(false); + dumpPartitionsInfo(cctx, log); + } CheckpointStatus status = readCheckpointStatus(); @@ -2002,8 +1902,11 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC true ); - // Restore state for all groups. - restorePartitionStates(cctx.cache().cacheGroups(), logicalState.partitionRecoveryStates); + if (recoveryVerboseLogging && log.isInfoEnabled()) { + log.info("Partition states information after LOGICAL RECOVERY phase:"); + + dumpPartitionsInfo(cctx, log); + } walTail = tailPointer(logicalState.lastRead); @@ -2043,32 +1946,6 @@ private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { return lastRead != null ? lastRead.next() : null; } - /** - * @param forGroups Cache groups. - * @param partitionStates Partition states. - * @throws IgniteCheckedException If failed. - */ - private void restorePartitionStates( - Collection forGroups, - Map partitionStates - ) throws IgniteCheckedException { - long startRestorePart = U.currentTimeMillis(); - - if (log.isInfoEnabled()) - log.info("Restoring partition state for local groups."); - - long totalProcessed = 0; - - for (CacheGroupContext grp : forGroups) - totalProcessed += grp.restorePartitionStates(partitionStates); - - if (log.isInfoEnabled()) - log.info("Finished restoring partition state for local groups [" + - "groupsProcessed" + forGroups.size() + - "partitionsProcessed=" + totalProcessed + - ", time=" + (U.currentTimeMillis() - startRestorePart) + "ms]"); - } - /** * Called when all partitions have been fully restored and pre-created on node start. * @@ -2096,7 +1973,7 @@ private void restorePartitionStates( * @throws IgniteCheckedException If failed. * @throws StorageException In case I/O error occurred during operations with storage. */ - private WALPointer performBinaryMemoryRestore( + private RestoreBinaryState performBinaryMemoryRestore( CheckpointStatus status, Predicate cacheGroupsPredicate, boolean finalizeState @@ -2245,7 +2122,7 @@ private WALPointer performBinaryMemoryRestore( if (!finalizeState) return null; - WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); + FileWALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); if (status.needRestoreMemory()) { if (restoreBinaryState.needApplyBinaryUpdate()) @@ -2262,7 +2139,10 @@ private WALPointer performBinaryMemoryRestore( cpHistory.initialize(retreiveHistory()); - return lastReadPtr != null ? lastReadPtr.next() : null; + // Move pointer position to the end of last read record. + restoreBinaryState.lastRead = lastReadPtr != null ? lastReadPtr.next() : lastReadPtr; + + return restoreBinaryState; } /** @@ -2299,39 +2179,40 @@ private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedE * @param it WalIterator. * @param recPredicate Wal record filter. * @param entryPredicate Entry filter. - * @param partitionRecoveryStates Partition to restore state. */ public void applyUpdatesOnRecovery( @Nullable WALIterator it, - IgnitePredicate> recPredicate, - IgnitePredicate entryPredicate, - Map partitionRecoveryStates + IgniteBiPredicate recPredicate, + IgnitePredicate entryPredicate ) throws IgniteCheckedException { + if (it == null) + return; + cctx.walState().runWithOutWAL(() -> { - if (it != null) { - while (it.hasNext()) { - IgniteBiTuple next = it.next(); + while (it.hasNext()) { + IgniteBiTuple next = it.next(); - WALRecord rec = next.get2(); + WALRecord rec = next.get2(); - if (!recPredicate.apply(next)) - break; + if (!recPredicate.apply(next.get1(), rec)) + break; switch (rec.type()) { + case DATA_RECORD: - checkpointReadLock(); + checkpointReadLock(); - try { - DataRecord dataRec = (DataRecord)rec; + try { + DataRecord dataRec = (DataRecord)rec; - for (DataEntry dataEntry : dataRec.writeEntries()) { - if (entryPredicate.apply(dataEntry)) { - checkpointReadLock(); + for (DataEntry dataEntry : dataRec.writeEntries()) { + if (entryPredicate.apply(dataEntry)) { + checkpointReadLock(); - try { - int cacheId = dataEntry.cacheId(); + try { + int cacheId = dataEntry.cacheId(); - GridCacheContext cacheCtx = cctx.cacheContext(cacheId); + GridCacheContext cacheCtx = cctx.cacheContext(cacheId); if (cacheCtx != null) applyUpdate(cacheCtx, dataEntry); @@ -2344,6 +2225,7 @@ else if (log != null) } } } + } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -2354,23 +2236,10 @@ else if (log != null) break; - default: - // Skip other records. - } + default: + // Skip other records. } } - - checkpointReadLock(); - - try { - restorePartitionStates(cctx.cache().cacheGroups(), partitionRecoveryStates); - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } - finally { - checkpointReadUnlock(); - } }); } @@ -2492,6 +2361,9 @@ private RestoreLogicalState applyLogicalUpdates( log.info("Finished applying WAL changes [updatesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + " ms]"); + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) + lsnr.afterLogicalUpdatesApplied(restoreLogicalState); + return restoreLogicalState; } @@ -4133,12 +4005,12 @@ public void walSegsCoveredRange(final IgniteBiTuple walSegsCoveredRa /** * */ - private static class CheckpointStatus { + public static class CheckpointStatus { /** Null checkpoint UUID. */ private static final UUID NULL_UUID = new UUID(0L, 0L); /** Null WAL pointer. */ - private static final WALPointer NULL_PTR = new FileWALPointer(0, 0, 0); + public static final WALPointer NULL_PTR = new FileWALPointer(0, 0, 0); /** */ private long cpStartTs; @@ -4602,6 +4474,97 @@ else if (key.startsWith(WAL_GLOBAL_KEY_PREFIX)) return null; } + /** + * Method dumps partitions info see {@link #dumpPartitionsInfo(CacheGroupContext, IgniteLogger)} + * for all persistent cache groups. + * + * @param cctx Shared context. + * @param log Logger. + * @throws IgniteCheckedException If failed. + */ + private static void dumpPartitionsInfo(GridCacheSharedContext cctx, IgniteLogger log) throws IgniteCheckedException { + for (CacheGroupContext grp : cctx.cache().cacheGroups()) { + if (grp.isLocal() || !grp.persistenceEnabled()) + continue; + + dumpPartitionsInfo(grp, log); + } + } + + /** + * Retrieves from page memory meta information about given {@code grp} group partitions + * and dumps this information to log INFO level. + * + * @param grp Cache group. + * @param log Logger. + * @throws IgniteCheckedException If failed. + */ + private static void dumpPartitionsInfo(CacheGroupContext grp, IgniteLogger log) throws IgniteCheckedException { + PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory(); + + IgnitePageStoreManager pageStore = grp.shared().pageStore(); + + assert pageStore != null : "Persistent cache should have initialize page store manager."; + + for (int p = 0; p < grp.affinity().partitions(); p++) { + if (!pageStore.exists(grp.groupId(), p)) + continue; + + pageStore.ensure(grp.groupId(), p); + + if (pageStore.pages(grp.groupId(), p) <= 1) { + log.info("Partition [id=" + p + ", state=N/A (only file header) ]"); + + continue; + } + + long partMetaId = pageMem.partitionMetaPageId(grp.groupId(), p); + long partMetaPage = pageMem.acquirePage(grp.groupId(), partMetaId); + + try { + long pageAddr = pageMem.readLock(grp.groupId(), partMetaId, partMetaPage); + + try { + PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr); + + GridDhtPartitionState partitionState = GridDhtPartitionState.fromOrdinal(io.getPartitionState(pageAddr)); + + String state = partitionState != null ? partitionState.toString() : "N/A"; + + long updateCounter = io.getUpdateCounter(pageAddr); + long size = io.getSize(pageAddr); + + log.info("Partition [grp=" + grp.cacheOrGroupName() + + ", id=" + p + + ", state=" + state + + ", counter=" + updateCounter + + ", size=" + size + "]"); + } + finally { + pageMem.readUnlock(grp.groupId(), partMetaId, partMetaPage); + } + } + finally { + pageMem.releasePage(grp.groupId(), partMetaId, partMetaPage); + } + } + } + + /** + * Recovery lifecycle for read-write metastorage. + */ + private class MetastorageRecoveryLifecycle implements DatabaseLifecycleListener { + @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { + cctx.pageStore().initializeForMetastorage(); + } + + @Override public void afterBinaryMemoryRestore(RestoreBinaryState binaryState) throws IgniteCheckedException { + assert metaStorage == null; + + metaStorage = createMetastorage(false); + } + } + /** * Abstract class for create restore context. */ @@ -4673,12 +4636,9 @@ public WALRecord next(WALIterator it) throws IgniteCheckedException { // Filter data entries by group id. List filteredEntries = dataRecord.writeEntries().stream() .filter(entry -> { - if (entry == null) - return false; - int cacheId = entry.cacheId(); - return cctx != null && cctx.cacheContext(cacheId) != null && cacheGroupPredicate.test(cctx.cacheContext(cacheId).groupId()); + return cctx.cacheContext(cacheId) != null && cacheGroupPredicate.test(cctx.cacheContext(cacheId).groupId()); }) .collect(Collectors.toList()); @@ -4714,7 +4674,7 @@ public WALRecord next(WALIterator it) throws IgniteCheckedException { * * @return Last read WAL record pointer. */ - public WALPointer lastReadRecordPointer() { + public FileWALPointer lastReadRecordPointer() { return lastRead; } @@ -4732,7 +4692,7 @@ public boolean throwsCRCError(){ /** * Restore memory context. Tracks the safety of binary recovery. */ - private class RestoreBinaryState extends RestoreStateContext { + public class RestoreBinaryState extends RestoreStateContext { /** Checkpoint status. */ private final CheckpointStatus status; @@ -4807,7 +4767,7 @@ public boolean needApplyBinaryUpdate() { /** * Restore logical state context. Tracks the safety of logical recovery. */ - private class RestoreLogicalState extends RestoreStateContext { + public class RestoreLogicalState extends RestoreStateContext { /** States of partitions recovered during applying logical updates. */ private final Map partitionRecoveryStates = new HashMap<>(); @@ -4817,6 +4777,13 @@ private class RestoreLogicalState extends RestoreStateContext { public RestoreLogicalState(long lastArchivedSegment, Predicate cacheGroupsPredicate) { super(lastArchivedSegment, cacheGroupsPredicate, false); } + + /** + * @return Map of restored partition states for cache groups. + */ + public Map partitionRecoveryStates() { + return Collections.unmodifiableMap(partitionRecoveryStates); + } } /** Indicates checkpoint read lock acquisition failure which did not lead to node invalidation. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 5f6511dcfd046..7e33de11ba05c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -71,6 +71,7 @@ import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; import org.apache.ignite.internal.processors.cache.persistence.partstate.PagesAllocationRange; import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO; @@ -112,6 +113,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple /** */ private ReuseListImpl reuseList; + /** Flag indicates that all group partitions have restored their state from page memory / disk. */ + private volatile boolean partitionStatesRestored; + /** {@inheritDoc} */ @Override protected void initPendingTree(GridCacheContext cctx) throws IgniteCheckedException { // No-op. Per-partition PendingTree should be used. @@ -396,6 +400,142 @@ else if (needSnapshot) tryAddEmptyPartitionToSnapshot(store, ctx); } + /** {@inheritDoc} */ + @Override public long restorePartitionStates(Map partitionRecoveryStates) throws IgniteCheckedException { + if (grp.isLocal() || !grp.affinityNode() || !grp.dataRegion().config().isPersistenceEnabled()) + return 0; + + if (partitionStatesRestored) + return 0; + + long processed = 0; + + PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory(); + + for (int p = 0; p < grp.affinity().partitions(); p++) { + PartitionRecoverState recoverState = partitionRecoveryStates.get(new GroupPartitionId(grp.groupId(), p)); + + if (ctx.pageStore().exists(grp.groupId(), p)) { + ctx.pageStore().ensure(grp.groupId(), p); + + if (ctx.pageStore().pages(grp.groupId(), p) <= 1) { + if (log.isDebugEnabled()) + log.debug("Skipping partition on recovery (pages less than 1) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + "]"); + + continue; + } + + if (log.isDebugEnabled()) + log.debug("Creating partition on recovery (exists in page store) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + "]"); + + processed++; + + GridDhtLocalPartition part = grp.topology().forceCreatePartition(p); + + onPartitionInitialCounterUpdated(p, 0); + + ctx.database().checkpointReadLock(); + + try { + long partMetaId = pageMem.partitionMetaPageId(grp.groupId(), p); + long partMetaPage = pageMem.acquirePage(grp.groupId(), partMetaId); + + try { + long pageAddr = pageMem.writeLock(grp.groupId(), partMetaId, partMetaPage); + + boolean changed = false; + + try { + PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr); + + if (recoverState != null) { + io.setPartitionState(pageAddr, (byte) recoverState.stateId()); + + changed = updateState(part, recoverState.stateId()); + + if (recoverState.stateId() == GridDhtPartitionState.OWNING.ordinal() + || (recoverState.stateId() == GridDhtPartitionState.MOVING.ordinal() + && part.initialUpdateCounter() < recoverState.updateCounter())) { + part.initialUpdateCounter(recoverState.updateCounter()); + + changed = true; + } + + if (log.isDebugEnabled()) + log.debug("Restored partition state (from WAL) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + + ", updCntr=" + part.initialUpdateCounter() + "]"); + } + else { + int stateId = (int) io.getPartitionState(pageAddr); + + changed = updateState(part, stateId); + + if (log.isDebugEnabled()) + log.debug("Restored partition state (from page memory) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + + ", updCntr=" + part.initialUpdateCounter() + ", stateId=" + stateId + "]"); + } + } + finally { + pageMem.writeUnlock(grp.groupId(), partMetaId, partMetaPage, null, changed); + } + } + finally { + pageMem.releasePage(grp.groupId(), partMetaId, partMetaPage); + } + } + finally { + ctx.database().checkpointReadUnlock(); + } + } + else if (recoverState != null) { + GridDhtLocalPartition part = grp.topology().forceCreatePartition(p); + + onPartitionInitialCounterUpdated(p, recoverState.updateCounter()); + + updateState(part, recoverState.stateId()); + + processed++; + + if (log.isDebugEnabled()) + log.debug("Restored partition state (from WAL) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + ", state=" + part.state() + + ", updCntr=" + part.initialUpdateCounter() + "]"); + } + else { + if (log.isDebugEnabled()) + log.debug("Skipping partition on recovery (no page store OR wal state) " + + "[grp=" + grp.cacheOrGroupName() + ", p=" + p + "]"); + } + } + + partitionStatesRestored = true; + + return processed; + } + + /** + * @param part Partition to restore state for. + * @param stateId State enum ordinal. + * @return Updated flag. + */ + private boolean updateState(GridDhtLocalPartition part, int stateId) { + if (stateId != -1) { + GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId); + + assert state != null; + + part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state); + + return true; + } + + return false; + } + /** * Check that we need to snapshot this partition and add it to map. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 4966bcaa0290d..7fc70d0b8923d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -696,22 +696,12 @@ public DataStorageMetrics persistentStoreMetrics() { return null; } - /** - * Creates file with current timestamp and specific "node-started.bin" suffix - * and writes into memory recovery pointer. - * - * @param ptr Memory recovery wal pointer. - */ - public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException { - // No-op. - } - /** * @param memPlcName Name of {@link DataRegion} to obtain {@link DataRegionMetrics} for. * @return {@link DataRegionMetrics} snapshot for specified {@link DataRegion} or {@code null} if * no {@link DataRegion} is configured for specified name. */ - @Nullable public DataRegionMetrics memoryMetrics(String memPlcName) { + public @Nullable DataRegionMetrics memoryMetrics(String memPlcName) { if (!F.isEmpty(memMetricsMap)) { DataRegionMetrics memMetrics = memMetricsMap.get(memPlcName); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java index 6ea7e002b6e6d..5e5917871983f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java @@ -80,7 +80,7 @@ public void length(int len) { } /** {@inheritDoc} */ - @Override public WALPointer next() { + @Override public FileWALPointer next() { if (len == 0) throw new IllegalStateException("Failed to calculate next WAL pointer " + "(this pointer is a terminal): " + this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index ee40039994548..579c06eb80b49 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -221,7 +221,7 @@ public RecordDataV1Serializer(GridCacheSharedContext cctx) { * @return {@code True} if this record should be encrypted. */ private boolean needEncryption(WALRecord rec) { - if (!(rec instanceof WalRecordCacheGroupAware) || rec instanceof MetastoreDataRecord) + if (!(rec instanceof WalRecordCacheGroupAware)) return false; return needEncryption(((WalRecordCacheGroupAware)rec).groupId()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java index 425f9b955a31d..93cc074e6d6ab 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java @@ -21,21 +21,24 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.OpenOption; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; +import java.util.stream.Collectors; import com.google.common.collect.Lists; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -45,12 +48,17 @@ import org.apache.ignite.failure.StopNodeFailureHandler; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCacheUtils; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; -import org.apache.ignite.internal.processors.cache.persistence.wal.memtracker.PageMemoryTrackerConfiguration; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -94,7 +102,6 @@ public class IgniteLogicalRecoveryTest extends GridCommonAbstractTest { ); DataStorageConfiguration dsCfg = new DataStorageConfiguration() - .setAlwaysWriteFullPages(true) .setWalMode(WALMode.LOG_ONLY) .setCheckpointFrequency(1024 * 1024 * 1024) // Disable automatic checkpoints. .setDefaultDataRegionConfiguration( @@ -110,7 +117,11 @@ public class IgniteLogicalRecoveryTest extends GridCommonAbstractTest { if (ioFactory != null) dsCfg.setFileIOFactory(ioFactory); - cfg.setPluginConfigurations(new PageMemoryTrackerConfiguration().setEnabled(false).setCheckPagesOnCheckpoint(true)); + TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi(); + + spi.record(GridDhtPartitionDemandMessage.class); + + cfg.setCommunicationSpi(spi); return cfg; } @@ -130,16 +141,16 @@ private CacheConfiguration cacheConfiguration(String name, Cache * @param cacheMode Cache mode. * @param atomicityMode Atomicity mode. */ - private CacheConfiguration cacheConfiguration(String name, @Nullable String groupName, CacheMode cacheMode, CacheAtomicityMode atomicityMode) { - CacheConfiguration cfg = new CacheConfiguration<>(); + protected CacheConfiguration cacheConfiguration(String name, @Nullable String groupName, CacheMode cacheMode, CacheAtomicityMode atomicityMode) { + CacheConfiguration cfg = new CacheConfiguration<>(name) + .setGroupName(groupName) + .setCacheMode(cacheMode) + .setAtomicityMode(atomicityMode) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setBackups(2) + .setAffinity(new RendezvousAffinityFunction(false, 32)); - cfg.setGroupName(groupName); - cfg.setName(name); - cfg.setCacheMode(cacheMode); - cfg.setAtomicityMode(atomicityMode); - cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - cfg.setBackups(2); - cfg.setAffinity(new RendezvousAffinityFunction(false, 32)); + cfg.setIndexedTypes(Integer.class, Integer.class); return cfg; } @@ -172,25 +183,23 @@ public void testRecoveryOnJoinToActiveCluster() throws Exception { IgniteEx node = grid(2); - AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); - - aggCacheLoader.start(); + AggregateCacheLoader cacheLoader = new AggregateCacheLoader(node); - U.sleep(3000); + cacheLoader.loadByTime(5_000).get(); forceCheckpoint(); - U.sleep(3000); - - aggCacheLoader.stop(); + cacheLoader.loadByTime(5_000).get(); stopGrid(2, true); - startGrid(2); + node = startGrid(2); awaitPartitionMapExchange(); - aggCacheLoader.consistencyCheck(grid(2)); + cacheLoader.consistencyCheck(node); + + checkNoRebalanceAfterRecovery(); } /** @@ -203,29 +212,27 @@ public void testRecoveryOnJoinToInactiveCluster() throws Exception { IgniteEx node = grid(2); - AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); - - aggCacheLoader.start(); + AggregateCacheLoader cacheLoader = new AggregateCacheLoader(node); - U.sleep(3000); + cacheLoader.loadByTime(5_000).get(); forceCheckpoint(); - U.sleep(3000); - - aggCacheLoader.stop(); + cacheLoader.loadByTime(5_000).get(); stopGrid(2, true); crd.cluster().active(false); - startGrid(2); + node = startGrid(2); crd.cluster().active(true); awaitPartitionMapExchange(); - aggCacheLoader.consistencyCheck(grid(2)); + checkNoRebalanceAfterRecovery(); + + cacheLoader.consistencyCheck(node); } /** @@ -268,17 +275,13 @@ private void doTestWithDynamicCaches(List dynamicCaches) thr node.getOrCreateCaches(dynamicCaches); - AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); - - aggCacheLoader.start(); + AggregateCacheLoader cacheLoader = new AggregateCacheLoader(node); - U.sleep(3000); + cacheLoader.loadByTime(5_000).get(); forceCheckpoint(); - U.sleep(3000); - - aggCacheLoader.stop(); + cacheLoader.loadByTime(5_000).get(); stopGrid(2, true); @@ -286,8 +289,10 @@ private void doTestWithDynamicCaches(List dynamicCaches) thr awaitPartitionMapExchange(); + checkNoRebalanceAfterRecovery(); + for (int idx = 0; idx < 3; idx++) - aggCacheLoader.consistencyCheck(grid(idx)); + cacheLoader.consistencyCheck(grid(idx)); } /** @@ -300,17 +305,13 @@ public void testRecoveryOnJoinToDifferentBlt() throws Exception { IgniteEx node = grid(2); - AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); + AggregateCacheLoader cacheLoader = new AggregateCacheLoader(node); - aggCacheLoader.start(); - - U.sleep(3000); + cacheLoader.loadByTime(5_000).get(); forceCheckpoint(); - U.sleep(3000); - - aggCacheLoader.stop(); + cacheLoader.loadByTime(5_000).get(); stopGrid(2, true); @@ -323,34 +324,26 @@ public void testRecoveryOnJoinToDifferentBlt() throws Exception { awaitPartitionMapExchange(); for (int idx = 0; idx < 3; idx++) - aggCacheLoader.consistencyCheck(grid(idx)); + cacheLoader.consistencyCheck(grid(idx)); } /** * */ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { - // Crash recovery fails because of the bug in pages recycling. - // Test passes if don't perform removes in cache loader. - fail("https://issues.apache.org/jira/browse/IGNITE-9303"); - IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3, false); crd.cluster().active(true); IgniteEx node = grid(2); - AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node); - - aggCacheLoader.start(); + AggregateCacheLoader cacheLoader = new AggregateCacheLoader(node); - U.sleep(3000); + cacheLoader.loadByTime(5_000).get(); forceCheckpoint(); - U.sleep(3000); - - aggCacheLoader.stop(); + cacheLoader.loadByTime(5_000).get(); stopGrid(2, false); @@ -363,6 +356,18 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { } catch (Exception expected) { } + // Wait until node will leave cluster. + GridTestUtils.waitForCondition(() -> { + try { + grid(2); + } + catch (IgniteIllegalStateException e) { + return true; + } + + return false; + }, getTestTimeout()); + ioFactory = null; // Start node again and check recovery. @@ -370,8 +375,10 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { awaitPartitionMapExchange(); + checkNoRebalanceAfterRecovery(); + for (int idx = 0; idx < 3; idx++) - aggCacheLoader.consistencyCheck(grid(idx)); + cacheLoader.consistencyCheck(grid(idx)); } /** {@inheritDoc} */ @@ -381,7 +388,30 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { /** {@inheritDoc} */ @Override protected long getTestTimeout() { - return 600 * 1000; + return 120 * 1000; + } + + /** + * Method checks that there were no rebalance for all caches (excluding sys cache). + */ + private void checkNoRebalanceAfterRecovery() { + int sysCacheGroupId = CU.cacheId(GridCacheUtils.UTILITY_CACHE_NAME); + + List nodes = G.allGrids(); + + for (Ignite node : nodes) { + TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(node); + + List rebalancedGroups = spi.recordedMessages(true).stream() + .map(msg -> (GridDhtPartitionDemandMessage) msg) + .map(msg -> msg.groupId()) + .filter(grpId -> grpId != sysCacheGroupId) + .distinct() + .collect(Collectors.toList()); + + Assert.assertTrue("There was unexpected rebalance for some groups" + + " [node=" + node.name() + ", groups=" + rebalancedGroups + ']', rebalancedGroups.isEmpty()); + } } /** @@ -389,59 +419,50 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { */ private static class AggregateCacheLoader { /** Ignite. */ - IgniteEx ignite; - - /** Stop flag. */ - AtomicBoolean stopFlag; + final IgniteEx ignite; /** Cache loaders. */ - Map cacheLoaders; + final List cacheLoaders; /** * @param ignite Ignite. */ public AggregateCacheLoader(IgniteEx ignite) { this.ignite = ignite; + + List cacheLoaders = new ArrayList<>(); + + for (String cacheName : ignite.cacheNames()) + cacheLoaders.add(new CacheLoader(ignite, cacheName)); + + this.cacheLoaders = cacheLoaders; } /** - * + * @param timeMillis Loading time in milliseconds. */ - public void start() { - if (stopFlag != null && !stopFlag.get()) - throw new IllegalStateException("Cache loaders must be stopped before start again"); - - stopFlag = new AtomicBoolean(); - cacheLoaders = new HashMap<>(); + public IgniteInternalFuture loadByTime(int timeMillis) { + GridCompoundFuture loadFut = new GridCompoundFuture(); - for (String cacheName : ignite.cacheNames()) { - CacheLoader loader = new CacheLoader(ignite, stopFlag, cacheName); + for (CacheLoader cacheLoader : cacheLoaders) { + long endTime = U.currentTimeMillis() + timeMillis; - IgniteInternalFuture loadFuture = GridTestUtils.runAsync(loader); + cacheLoader.stopPredicate = it -> U.currentTimeMillis() >= endTime; - cacheLoaders.put(loader, loadFuture); + loadFut.add(GridTestUtils.runAsync(cacheLoader)); } - } - /** - * - */ - public void stop() throws IgniteCheckedException { - if (stopFlag != null) - stopFlag.set(true); + loadFut.markInitialized(); - if (cacheLoaders != null) - for (IgniteInternalFuture loadFuture : cacheLoaders.values()) - loadFuture.get(); + return loadFut; } /** - * @param ignite Ignite. + * @param ignite Ignite node to check consistency from. */ public void consistencyCheck(IgniteEx ignite) { - if (cacheLoaders != null) - for (CacheLoader cacheLoader : cacheLoaders.keySet()) - cacheLoader.consistencyCheck(ignite); + for (CacheLoader cacheLoader : cacheLoaders) + cacheLoader.consistencyCheck(ignite); } } @@ -455,38 +476,38 @@ static class CacheLoader implements Runnable { /** Ignite. */ final IgniteEx ignite; - /** Stop flag. */ - final AtomicBoolean stopFlag; + /** Stop predicate. */ + volatile Predicate stopPredicate; /** Cache name. */ final String cacheName; /** Local cache. */ - final Map locCache = new ConcurrentHashMap<>(); + final Map locCache = new ConcurrentHashMap<>(); /** * @param ignite Ignite. - * @param stopFlag Stop flag. * @param cacheName Cache name. */ - public CacheLoader(IgniteEx ignite, AtomicBoolean stopFlag, String cacheName) { + public CacheLoader(IgniteEx ignite, String cacheName) { this.ignite = ignite; - this.stopFlag = stopFlag; this.cacheName = cacheName; } /** {@inheritDoc} */ @Override public void run() { - while (!stopFlag.get()) { + final Predicate predicate = stopPredicate; + + while (!predicate.test(ignite)) { ThreadLocalRandom rnd = ThreadLocalRandom.current(); int key = rnd.nextInt(KEYS_SPACE); boolean remove = rnd.nextInt(100) <= 20; - IgniteCache cache = ignite.getOrCreateCache(cacheName); - try { + IgniteCache cache = ignite.getOrCreateCache(cacheName); + if (remove) { cache.remove(key); @@ -496,10 +517,15 @@ public CacheLoader(IgniteEx ignite, AtomicBoolean stopFlag, String cacheName) { int[] payload = new int[KEYS_SPACE]; Arrays.fill(payload, key); - cache.put(key, payload); + TestValue val = new TestValue(key, payload); - locCache.put(key, payload); + cache.put(key, val); + + locCache.put(key, val); } + + // Throttle against GC. + U.sleep(1); } catch (Exception ignored) { } } @@ -509,14 +535,14 @@ public CacheLoader(IgniteEx ignite, AtomicBoolean stopFlag, String cacheName) { * */ public void consistencyCheck(IgniteEx ignite) { - IgniteCache cache = ignite.getOrCreateCache(cacheName); + IgniteCache cache = ignite.getOrCreateCache(cacheName); for (int key = 0; key < KEYS_SPACE; key++) { - int[] expectedValue = (int[]) locCache.get(key); - int[] actualValue = (int[]) cache.get(key); + TestValue expectedVal = locCache.get(key); + TestValue actualVal = cache.get(key); Assert.assertEquals("Consistency check failed for: " + cache.getName() + ", key=" + key, - arrayToString(expectedValue), arrayToString(actualValue)); + expectedVal, actualVal); } } @@ -566,21 +592,45 @@ static class CheckpointFailIoFactory implements FileIOFactory { } /** - * @param arr Array. + * Test payload with indexed field. */ - static String arrayToString(int[] arr) { - if (arr == null) - return "null"; + static class TestValue { + /** Indexed field. */ + @QuerySqlField(index = true) + private final int indexedField; - StringBuilder sb = new StringBuilder(); + /** Payload. */ + private final int[] payload; - sb.append('['); + /** + * @param indexedField Indexed field. + * @param payload Payload. + */ + public TestValue(int indexedField, int[] payload) { + this.indexedField = indexedField; + this.payload = payload; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + TestValue testValue = (TestValue) o; + + return indexedField == testValue.indexedField && + Arrays.equals(payload, testValue.payload); + } - for (int i = 0; i < Math.min(arr.length, 10); i++) - sb.append(i).append(","); + /** {@inheritDoc} */ + @Override public int hashCode() { + int result = Objects.hash(indexedField); - sb.append(']'); + result = 31 * result + Arrays.hashCode(payload); - return sb.toString(); + return result; + } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java index 43302855dc541..8d3153d65a201 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java @@ -160,64 +160,6 @@ public void testRecoveringOnCacheInitFail() throws Exception { grid.cluster().active(true); } - /** - * Test node stopping & recovering on start marker writing fail during activation. - * - * @throws Exception If test failed. - */ - public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception { - // Fail to write node start marker tmp file at the second checkpoint. Pass only initial checkpoint. - ioFactory = new FilteringFileIOFactory("started.bin" + FilePageStoreManager.TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20)); - - IgniteEx grid = startGrid(0); - grid.cluster().active(true); - - for (int i = 0; i < 1000; i++) { - byte payload = (byte) i; - byte[] data = new byte[2048]; - Arrays.fill(data, payload); - - grid.cache(CACHE_NAME).put(i, data); - } - - stopAllGrids(); - - boolean activationFailed = false; - try { - grid = startGrid(0); - } - catch (IgniteCheckedException e) { - boolean interrupted = Thread.interrupted(); - - if (interrupted) - log.warning("Ignore interrupted excpetion [" + - "thread=" + Thread.currentThread().getName() + ']', e); - - activationFailed = true; - } - - Assert.assertTrue("Ignite instance startup must be failed", activationFailed); - - // Grid should be automatically stopped after checkpoint fail. - awaitStop(grid); - - // Grid should be successfully recovered after stopping. - ioFactory = null; - - IgniteEx recoveredGrid = startGrid(0); - recoveredGrid.cluster().active(true); - - for (int i = 0; i < 1000; i++) { - byte payload = (byte) i; - byte[] data = new byte[2048]; - Arrays.fill(data, payload); - - byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i); - Assert.assertArrayEquals(data, actualData); - } - } - - /** * Test node stopping & recovering on checkpoint begin fail. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java index 906f19199a4b7..1a1d44909cf5b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java @@ -23,7 +23,6 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashMap; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; @@ -178,7 +177,7 @@ private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exce boolean fail = false; try (WALIterator it = sharedContext.wal().replay(null)) { - dbMgr.applyUpdatesOnRecovery(it, (tup) -> true, (entry) -> true, new HashMap<>()); + dbMgr.applyUpdatesOnRecovery(it, (ptr, rec) -> true, (entry) -> true); } catch (IgniteCheckedException e) { if (nodeStopPoint.needCleanUp) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java index 3374860baa7c1..5bf7e7f908c11 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java @@ -387,13 +387,12 @@ public void testSeekingStartInCompactedSegment() throws Exception { File[] cpMarkers = cpMarkersDir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !( - name.equals(cpMarkersToSave[0].getName()) || - name.equals(cpMarkersToSave[1].getName()) || - name.equals(cpMarkersToSave[2].getName()) || - name.equals(cpMarkersToSave[3].getName()) || - name.equals(cpMarkersToSave[4].getName()) - ); + for (File cpMarker : cpMarkersToSave) { + if (cpMarker.getName().equals(name)) + return false; + } + + return true; } }); From aa30082042087d8efec1a73565ec390893602407 Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 15 Oct 2018 17:58:59 +0300 Subject: [PATCH 109/403] Log information about lost partitions. --- .../dht/topology/GridDhtPartitionTopologyImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 2e555f44bc77d..9d6eb9e0d27d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -2047,6 +2047,12 @@ private void rebuildDiff(AffinityAssignment affAssignment) { } } + if (!recentlyLost.isEmpty()) { + U.warn(log, "Detected lost partitions [grp=" + grp.cacheOrGroupName() + + ", parts=" + S.compact(recentlyLost) + + ", plc=" + plc + "]"); + } + // Update partition state on all nodes. for (Integer part : lost) { long updSeq = updateSeq.incrementAndGet(); From ebbffda57359889fcbcee48785aac1e0d2f50dd7 Mon Sep 17 00:00:00 2001 From: Semyon Boikov Date: Tue, 20 Nov 2018 17:05:01 +0300 Subject: [PATCH 110/403] IGNITE-10043 Do not reset the list of LOST partitions when only one server node is left in the cluster - Fixes #5436 --- .../GridDhtPartitionsExchangeFuture.java | 5 +- .../topology/GridDhtPartitionTopology.java | 5 +- .../GridDhtPartitionTopologyImpl.java | 115 ++++++++---------- ...gniteCachePartitionLossPolicySelfTest.java | 6 +- 4 files changed, 57 insertions(+), 74 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 64eadfcdb6cd9..c3c3386f442bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3004,7 +3004,10 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { for (CacheGroupContext grp : cctx.cache().cacheGroups()) { if (!grp.isLocal()) { - boolean detectedOnGrp = grp.topology().detectLostPartitions(resTopVer, events().lastEvent()); + // Do not trigger lost partition events on start. + boolean event = !localJoinExchange() && !activateCluster(); + + boolean detectedOnGrp = grp.topology().detectLostPartitions(resTopVer, event ? events().lastEvent() : null); detected |= detectedOnGrp; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index 25b284ed29f46..bdf16da378144 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -25,6 +25,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.events.EventType; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.managers.discovery.DiscoCache; import org.apache.ignite.internal.processors.affinity.AffinityAssignment; @@ -327,10 +328,10 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, * This method should be called on topology coordinator after all partition messages are received. * * @param resTopVer Exchange result version. - * @param discoEvt Discovery event for which we detect lost partitions. + * @param discoEvt Discovery event for which we detect lost partitions if {@link EventType#EVT_CACHE_REBALANCE_PART_DATA_LOST} event should be fired. * @return {@code True} if partitions state got updated. */ - public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, DiscoveryEvent discoEvt); + public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, @Nullable DiscoveryEvent discoEvt); /** * Resets the state of all LOST partitions to OWNING. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 9d6eb9e0d27d2..aeb71e9243e27 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -110,8 +111,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology { /** Node to partition map. */ private GridDhtPartitionFullMap node2part; - /** Partitions map for left nodes. */ - private GridDhtPartitionFullMap leftNode2Part = new GridDhtPartitionFullMap(); + /** */ + private Set lostParts; /** */ private final Map> diffFromAffinity = new HashMap<>(); @@ -1466,13 +1467,6 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD log.trace("Removing left node from full map update [grp=" + grp.cacheOrGroupName() + ", nodeId=" + nodeId + ", partMap=" + partMap + ']'); - if (node2part.containsKey(nodeId)) { - GridDhtPartitionMap map = partMap.get(nodeId); - - if (map != null) - leftNode2Part.put(nodeId, map); - } - it.remove(); } } @@ -2003,7 +1997,7 @@ private void rebuildDiff(AffinityAssignment affAssignment) { } /** {@inheritDoc} */ - @Override public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, DiscoveryEvent discoEvt) { + @Override public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, @Nullable DiscoveryEvent discoEvt) { ctx.database().checkpointReadLock(); try { @@ -2013,48 +2007,55 @@ private void rebuildDiff(AffinityAssignment affAssignment) { if (node2part == null) return false; + PartitionLossPolicy plc = grp.config().getPartitionLossPolicy(); + + assert plc != null; + int parts = grp.affinity().partitions(); - Set lost = new HashSet<>(parts); + Set recentlyLost = null; - for (int p = 0; p < parts; p++) - lost.add(p); + boolean changed = false; - for (GridDhtPartitionMap partMap : node2part.values()) { - for (Map.Entry e : partMap.entrySet()) { - if (e.getValue() == OWNING) { - lost.remove(e.getKey()); + for (int part = 0; part < parts; part++) { + boolean lost = F.contains(lostParts, part); - if (lost.isEmpty()) + if (!lost) { + boolean hasOwner = false; + + for (GridDhtPartitionMap partMap : node2part.values()) { + if (partMap.get(part) == OWNING) { + hasOwner = true; break; + } } - } - } - boolean changed = false; + if (!hasOwner) { + lost = true; - if (!F.isEmpty(lost)) { - PartitionLossPolicy plc = grp.config().getPartitionLossPolicy(); + if (lostParts == null) + lostParts = new TreeSet<>(); - assert plc != null; + lostParts.add(part); - Set recentlyLost = new HashSet<>(); + if (discoEvt != null) { + if (recentlyLost == null) + recentlyLost = new HashSet<>(); - for (Map.Entry leftEntry : leftNode2Part.entrySet()) { - for (Map.Entry entry : leftEntry.getValue().entrySet()) { - if (entry.getValue() == OWNING) - recentlyLost.add(entry.getKey()); - } - } + recentlyLost.add(part); - if (!recentlyLost.isEmpty()) { - U.warn(log, "Detected lost partitions [grp=" + grp.cacheOrGroupName() - + ", parts=" + S.compact(recentlyLost) - + ", plc=" + plc + "]"); + if (grp.eventRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST)) { + grp.addRebalanceEvent(part, + EVT_CACHE_REBALANCE_PART_DATA_LOST, + discoEvt.eventNode(), + discoEvt.type(), + discoEvt.timestamp()); + } + } + } } - // Update partition state on all nodes. - for (Integer part : lost) { + if (lost) { long updSeq = updateSeq.incrementAndGet(); GridDhtLocalPartition locPart = localPartition(part, resTopVer, false, true); @@ -2080,21 +2081,17 @@ else if (plc != PartitionLossPolicy.IGNORE) { e.getValue().put(part, LOST); } } - - if (recentlyLost.contains(part) && grp.eventRecordable(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST)) { - grp.addRebalanceEvent(part, - EVT_CACHE_REBALANCE_PART_DATA_LOST, - discoEvt.eventNode(), - discoEvt.type(), - discoEvt.timestamp()); - } } + } - if (plc != PartitionLossPolicy.IGNORE) - grp.needsRecovery(true); + if (recentlyLost != null) { + U.warn(log, "Detected lost partitions [grp=" + grp.cacheOrGroupName() + + ", parts=" + S.compact(recentlyLost) + + ", plc=" + plc + "]"); } - leftNode2Part.clear(); + if (lostParts != null && plc != PartitionLossPolicy.IGNORE) + grp.needsRecovery(true); return changed; } @@ -2144,6 +2141,8 @@ else if (plc != PartitionLossPolicy.IGNORE) { checkEvictions(updSeq, grp.affinity().readyAffinity(resTopVer)); + lostParts = null; + grp.needsRecovery(false); } finally { @@ -2163,22 +2162,7 @@ else if (plc != PartitionLossPolicy.IGNORE) { lock.readLock().lock(); try { - Set res = null; - - int parts = grp.affinity().partitions(); - - for (GridDhtPartitionMap partMap : node2part.values()) { - for (Map.Entry e : partMap.entrySet()) { - if (e.getValue() == LOST) { - if (res == null) - res = new HashSet<>(parts); - - res.add(e.getKey()); - } - } - } - - return res == null ? Collections.emptySet() : res; + return lostParts == null ? Collections.emptySet() : new HashSet<>(lostParts); } finally { lock.readLock().unlock(); @@ -2534,9 +2518,6 @@ private void removeNode(UUID nodeId) { GridDhtPartitionMap parts = node2part.remove(nodeId); - if (parts != null) - leftNode2Part.put(nodeId, parts); - if (!grp.isReplicated()) { if (parts != null) { for (Integer p : parts.keySet()) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java index cfe578db454a1..226ae229bec2a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java @@ -299,8 +299,6 @@ public void testReadWriteSafeWithBackupsAfterKillThreeNodes() throws Exception { * @throws Exception if failed. */ public void testReadWriteSafeWithBackupsAfterKillThreeNodesWithPersistence() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10043"); - partLossPlc = PartitionLossPolicy.READ_WRITE_SAFE; backups = 1; @@ -924,12 +922,12 @@ private List changeTopology() throws Exception { for (Map map : lostMap) { for (Map.Entry entry : map.entrySet()) - assertTrue("Failed to wait for partition LOST event for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); + assertTrue("Failed to wait for partition LOST event for partition: " + entry.getKey(), entry.getValue().tryAcquire(1)); } for (Map map : lostMap) { for (Map.Entry entry : map.entrySet()) - assertFalse("Partition LOST event raised twice for partition:" + entry.getKey(), entry.getValue().tryAcquire(1)); + assertFalse("Partition LOST event raised twice for partition: " + entry.getKey(), entry.getValue().tryAcquire(1)); } return parts; From f2e1bdf9aca7d7722d20f09b5ef92b8af538d2a3 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Tue, 20 Nov 2018 19:09:10 +0300 Subject: [PATCH 111/403] IGNITE-9999 Added verbose logging for node recovery (fixed NPE) --- .../persistence/wal/serializer/RecordDataV1Serializer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 579c06eb80b49..f3888729afc5c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.managers.encryption.GridEncryptionManager; import org.apache.ignite.spi.encryption.EncryptionSpi; import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.wal.record.CacheState; @@ -232,7 +233,9 @@ private boolean needEncryption(WALRecord rec) { * @return {@code True} if this record should be encrypted. */ private boolean needEncryption(int grpId) { - return cctx.kernalContext().encryption().groupKey(grpId) != null; + GridEncryptionManager encMgr = cctx.kernalContext().encryption(); + + return encMgr != null && encMgr.groupKey(grpId) != null; } /** From 64911396ffb7d873ffd393fa3b3cc031753fface Mon Sep 17 00:00:00 2001 From: EdShangGG Date: Tue, 20 Nov 2018 11:39:19 +0300 Subject: [PATCH 112/403] IGNITE-10328 Allow to destroy cache in code which were created via SQL - Fixes #5431. Signed-off-by: Alexey Goncharuk (cherry picked from commit c8fd687) --- .../internal/processors/cache/ClusterCachesInfo.java | 8 -------- .../cache/index/H2DynamicTableSelfTest.java | 11 ++--------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 6ab4e679518e3..6718cf3ef7341 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -671,14 +671,6 @@ else if (req.stop()) { continue; } - if (!req.sql() && desc.sql()) { - ctx.cache().completeCacheStartFuture(req, false, - new IgniteCheckedException("Only cache created with cache API may be removed with " + - "direct call to destroyCache [cacheName=" + req.cacheName() + ']')); - - continue; - } - DynamicCacheDescriptor old = registeredCaches.remove(req.cacheName()); if (req.restart()) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java index c97d93437b34f..68a97fdda1875 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java @@ -816,7 +816,7 @@ public void testDropNonDynamicTable() throws Exception { } /** - * Test that attempting to destroy via cache API a cache created via SQL yields an error. + * Test that attempting to destroy via cache API a cache created via SQL finishes successfully. * @throws Exception if failed. */ @SuppressWarnings("ThrowableResultOfMethodCallIgnored") @@ -825,14 +825,7 @@ public void testDestroyDynamicSqlCache() throws Exception { " \"name\" varchar, \"surname\" varchar, \"age\" int, PRIMARY KEY (\"id\", \"city\")) WITH " + "\"template=cache\""); - GridTestUtils.assertThrows(null, new Callable() { - @Override public Object call() throws Exception { - client().destroyCache(cacheName("Person")); - - return null; - } - }, CacheException.class, - "Only cache created with cache API may be removed with direct call to destroyCache"); + client().destroyCache(cacheName("Person")); } /** From 76030e188558b3518ac9230871db224fee66ae3a Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 29 Oct 2018 16:01:12 +0700 Subject: [PATCH 113/403] IGNITE-10031 REST: Added "caches" parameter to "top" command to include/exclude information about caches. (cherry picked from commit 594aac83c39eec1d12f2e9cef0ed9af35e9a9dba) --- .../JettyRestProcessorAbstractSelfTest.java | 41 +++++++++++++++++++ .../top/GridTopologyCommandHandler.java | 28 +++++++------ .../rest/request/GridRestTopologyRequest.java | 19 ++++++++- .../http/jetty/GridJettyRestHandler.java | 3 ++ .../app/modules/agent/AgentManager.service.js | 2 +- .../agent/handlers/ClusterListener.java | 1 + 6 files changed, 80 insertions(+), 14 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java index a972bc33f1999..703f50d65c5ef 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java @@ -1428,6 +1428,25 @@ public void testTopology() throws Exception { assertEquals(publicCache.getConfiguration(CacheConfiguration.class).getCacheMode(), cacheMode); } } + + // Test that caches not included. + ret = content(null, GridRestCommand.TOPOLOGY, + "attr", "false", + "mtr", "false", + "caches", "false" + ); + + info("Topology command result: " + ret); + + res = jsonResponse(ret); + + assertEquals(gridCount(), res.size()); + + for (JsonNode node : res) { + assertTrue(node.get("attributes").isNull()); + assertTrue(node.get("metrics").isNull()); + assertTrue(node.get("caches").isNull()); + } } /** @@ -1447,6 +1466,12 @@ public void testNode() throws Exception { assertTrue(res.get("attributes").isObject()); assertTrue(res.get("metrics").isObject()); + JsonNode caches = res.get("caches"); + + assertTrue(caches.isArray()); + assertFalse(caches.isNull()); + assertEquals(grid(0).context().cache().publicCaches().size(), caches.size()); + ret = content(null, GridRestCommand.NODE, "attr", "false", "mtr", "false", @@ -1472,6 +1497,22 @@ public void testNode() throws Exception { res = jsonResponse(ret); assertTrue(res.isNull()); + + // Check that caches not included. + ret = content(null, GridRestCommand.NODE, + "id", grid(0).localNode().id().toString(), + "attr", "false", + "mtr", "false", + "caches", "false" + ); + + info("Topology command result: " + ret); + + res = jsonResponse(ret); + + assertTrue(res.get("attributes").isNull()); + assertTrue(res.get("metrics").isNull()); + assertTrue(res.get("caches").isNull()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java index 0390936e77395..edcb3741bc331 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java @@ -97,6 +97,7 @@ public GridTopologyCommandHandler(GridKernalContext ctx) { boolean mtr = req0.includeMetrics(); boolean attr = req0.includeAttributes(); + boolean caches = req0.includeCaches(); switch (req.command()) { case TOPOLOGY: { @@ -107,7 +108,7 @@ public GridTopologyCommandHandler(GridKernalContext ctx) { new ArrayList<>(allNodes.size()); for (ClusterNode node : allNodes) - top.add(createNodeBean(node, mtr, attr)); + top.add(createNodeBean(node, mtr, attr, caches)); res.setResponse(top); @@ -143,7 +144,7 @@ public boolean apply(ClusterNode n) { }); if (node != null) - res.setResponse(createNodeBean(node, mtr, attr)); + res.setResponse(createNodeBean(node, mtr, attr, caches)); else res.setResponse(null); @@ -196,14 +197,15 @@ public GridClientCacheBean createCacheBean(CacheConfiguration ccfg) { } /** - * Creates node bean out of grid node. Notice that cache attribute is handled separately. + * Creates node bean out of cluster node. Notice that cache attribute is handled separately. * - * @param node Grid node. - * @param mtr {@code true} to add metrics. - * @param attr {@code true} to add attributes. + * @param node Cluster node. + * @param mtr Whether to include node metrics. + * @param attr Whether to include node attributes. + * @param caches Whether to include node caches. * @return Grid Node bean. */ - private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean attr) { + private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean attr, boolean caches) { assert node != null; GridClientNodeBean nodeBean = new GridClientNodeBean(); @@ -216,14 +218,16 @@ private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean nodeBean.setTcpAddresses(nonEmptyList(node.>attribute(ATTR_REST_TCP_ADDRS))); nodeBean.setTcpHostNames(nonEmptyList(node.>attribute(ATTR_REST_TCP_HOST_NAMES))); - Map nodeCaches = ctx.discovery().nodePublicCaches(node); + if (caches) { + Map nodeCaches = ctx.discovery().nodePublicCaches(node); - Collection caches = new ArrayList<>(nodeCaches.size()); + Collection cacheBeans = new ArrayList<>(nodeCaches.size()); - for (CacheConfiguration ccfg : nodeCaches.values()) - caches.add(createCacheBean(ccfg)); + for (CacheConfiguration ccfg : nodeCaches.values()) + cacheBeans.add(createCacheBean(ccfg)); - nodeBean.setCaches(caches); + nodeBean.setCaches(cacheBeans); + } if (mtr) { ClusterMetrics metrics = node.metrics(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java index b02836743f67b..fadd178945e2b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java @@ -36,6 +36,9 @@ public class GridRestTopologyRequest extends GridRestRequest { /** Include node attributes flag. */ private boolean includeAttrs; + /** Include caches flag. With default value for compatibility. */ + private boolean includeCaches = true; + /** * @return Include metrics flag. */ @@ -64,6 +67,20 @@ public void includeAttributes(boolean includeAttrs) { this.includeAttrs = includeAttrs; } + /** + * @return Include caches flag. + */ + public boolean includeCaches() { + return includeCaches; + } + + /** + * @param includeCaches Include caches flag. + */ + public void includeCaches(boolean includeCaches) { + this.includeCaches = includeCaches; + } + /** * @return Node identifier, if specified, {@code null} otherwise. */ @@ -96,4 +113,4 @@ public void nodeIp(String nodeIp) { @Override public String toString() { return S.toString(GridRestTopologyRequest.class, this, super.toString()); } -} \ No newline at end of file +} diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java index 03bed694017e3..c0a4fe1b5c30a 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java +++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java @@ -689,6 +689,9 @@ private Object convert(String type, Object obj) throws IgniteCheckedException { restReq0.includeMetrics(Boolean.parseBoolean((String)params.get("mtr"))); restReq0.includeAttributes(Boolean.parseBoolean((String)params.get("attr"))); + String caches = (String)params.get("caches"); + restReq0.includeCaches(caches == null || Boolean.parseBoolean(caches)); + restReq0.nodeIp((String)params.get("ip")); restReq0.nodeId(uuidValue("id", params)); diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index f667374bbec2f..8d2ced2ad8a0e 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -545,7 +545,7 @@ export default class AgentManager { * @returns {Promise} */ topology(attr = false, mtr = false) { - return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr}); + return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches: false}); } /** diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java index 1c93798b28819..779b458b61cae 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java @@ -410,6 +410,7 @@ private RestResult topology(boolean full) throws IOException { params.put("cmd", "top"); params.put("attr", true); params.put("mtr", full); + params.put("caches", false); return restCommand(params); } From d11294b2d9f33ea4b6b41f01c2cf4870254bfbcf Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 4 Oct 2018 22:41:38 +0700 Subject: [PATCH 114/403] IGNITE-9795 Web Agent: Improved information message in case REST command failed on cluster (cherry picked from commit c59791196a6a54beada115b676feafb42bec53c1) --- .../java/org/apache/ignite/console/agent/rest/RestExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java index bb06c32579b86..737b966332edd 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java @@ -136,7 +136,7 @@ private RestResult parseResponse(Response res) throws IOException { if (res.code() == 404) return RestResult.fail(STATUS_FAILED, "Failed connect to cluster."); - return RestResult.fail(STATUS_FAILED, "Failed to execute REST command: " + res.message()); + return RestResult.fail(STATUS_FAILED, "Failed to execute REST command: " + res); } /** */ From e6f46dcda2bfa0f87819cf95e283662d03a7bc80 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Mon, 8 Oct 2018 09:47:00 +0700 Subject: [PATCH 115/403] IGNITE-9807 Web Agent: Added support for cluster ID. (cherry picked from commit 08fff5e71309d4c6bdf295f0d6bc041a30753de5) --- .../web-console/backend/app/agentsHandler.js | 3 ++- .../app/modules/agent/AgentManager.service.js | 4 ---- .../agent/handlers/ClusterListener.java | 20 +++++++++++++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/modules/web-console/backend/app/agentsHandler.js b/modules/web-console/backend/app/agentsHandler.js index 6d8c621474fb4..8d72dedd1f6f4 100644 --- a/modules/web-console/backend/app/agentsHandler.js +++ b/modules/web-console/backend/app/agentsHandler.js @@ -87,7 +87,8 @@ module.exports.factory = function(settings, mongo, AgentSocket) { constructor(top) { const clusterName = top.clusterName; - this.id = _.isEmpty(clusterName) ? `Cluster ${uuid().substring(0, 8).toUpperCase()}` : clusterName; + this.id = _.isEmpty(top.clusterId) ? uuid() : top.clusterId; + this.name = _.isEmpty(clusterName) ? `Cluster ${this.id.substring(0, 8).toUpperCase()}` : clusterName; this.nids = top.nids; this.addresses = top.addresses; this.clients = top.clients; diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index 8d2ced2ad8a0e..aa949a2d7a41d 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -73,10 +73,6 @@ class ConnectionState { } update(demo, count, clusters) { - _.forEach(clusters, (cluster) => { - cluster.name = cluster.id; - }); - this.clusters = clusters; if (_.isEmpty(this.clusters)) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java index 779b458b61cae..6985837e1044e 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java @@ -74,6 +74,9 @@ public class ClusterListener implements AutoCloseable { /** */ private static final IgniteProductVersion IGNITE_2_3 = IgniteProductVersion.fromString("2.3.0"); + /** Optional Ignite cluster ID. */ + public static final String IGNITE_CLUSTER_ID = "IGNITE_CLUSTER_ID"; + /** Unique Visor key to get events last order. */ private static final String EVT_LAST_ORDER_KEY = "WEB_AGENT_" + UUID.randomUUID().toString(); @@ -188,6 +191,9 @@ public void watch() { /** */ private static class TopologySnapshot { + /** */ + private String clusterId; + /** */ private String clusterName; @@ -242,6 +248,9 @@ private static T attribute(Map attrs, String name) { Map attrs = node.getAttributes(); + if (F.isEmpty(clusterId)) + clusterId = attribute(attrs, IGNITE_CLUSTER_ID); + if (F.isEmpty(clusterName)) clusterName = attribute(attrs, IGNITE_CLUSTER_NAME); @@ -268,6 +277,13 @@ private static T attribute(Map attrs, String name) { } } + /** + * @return Cluster id. + */ + public String getClusterId() { + return clusterId; + } + /** * @return Cluster name. */ @@ -384,11 +400,11 @@ else if (!F.isEmpty(cfg.nodeLogin()) && !F.isEmpty(cfg.nodePassword())) { sesTok = res.getSessionToken(); return res; - + case STATUS_FAILED: if (res.getError().startsWith(EXPIRED_SES_ERROR_MSG)) { sesTok = null; - + params.remove("sessionToken"); return restCommand(params); From 8d1c07ee4f4c5028767bce68db4eb42f6d33619c Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Mon, 8 Oct 2018 10:00:43 +0700 Subject: [PATCH 116/403] IGNITE-9808 Web Console: Refactored sockets caching. (cherry picked from commit 76daf05fef1392bb77c3be77c799df28dbaea7a1) --- .../web-console/backend/app/agentSocket.js | 20 ++- .../web-console/backend/app/agentsHandler.js | 142 ++++++++---------- .../backend/app/browsersHandler.js | 51 +++---- modules/web-console/backend/app/configure.js | 22 ++- modules/web-console/backend/app/settings.js | 3 +- 5 files changed, 113 insertions(+), 125 deletions(-) diff --git a/modules/web-console/backend/app/agentSocket.js b/modules/web-console/backend/app/agentSocket.js index aff62c4b3f304..0c7e6b2f78b9d 100644 --- a/modules/web-console/backend/app/agentSocket.js +++ b/modules/web-console/backend/app/agentSocket.js @@ -38,21 +38,33 @@ module.exports.factory = function() { class AgentSocket { /** * @param {Socket} socket Socket for interaction. + * @param {Object} accounts Active accounts. * @param {Array.} tokens Agent tokens. * @param {String} demoEnabled Demo enabled. */ - constructor(socket, tokens, demoEnabled) { + constructor(socket, accounts, tokens, demoEnabled) { Object.assign(this, { - socket, - tokens, + accounts, cluster: null, demo: { enabled: demoEnabled, browserSockets: [] - } + }, + socket, + tokens }); } + resetToken(oldToken) { + _.pull(this.tokens, oldToken); + + this.emitEvent('agent:reset:token', oldToken) + .then(() => { + if (_.isEmpty(this.tokens) && this.socket.connected) + this.socket.close(); + }); + } + /** * Send event to agent. * diff --git a/modules/web-console/backend/app/agentsHandler.js b/modules/web-console/backend/app/agentsHandler.js index 8d72dedd1f6f4..fd55ac3491247 100644 --- a/modules/web-console/backend/app/agentsHandler.js +++ b/modules/web-console/backend/app/agentsHandler.js @@ -50,22 +50,22 @@ module.exports.factory = function(settings, mongo, AgentSocket) { this.sockets = new Map(); } - get(token) { - let sockets = this.sockets.get(token); + get(account) { + let sockets = this.sockets.get(account._id.toString()); if (_.isEmpty(sockets)) - this.sockets.set(token, sockets = []); + this.sockets.set(account._id.toString(), sockets = []); return sockets; } /** * @param {AgentSocket} sock - * @param {String} token + * @param {String} account * @return {Array.} */ - add(token, sock) { - const sockets = this.get(token); + add(account, sock) { + const sockets = this.get(account); sockets.push(sock); } @@ -75,9 +75,9 @@ module.exports.factory = function(settings, mongo, AgentSocket) { * @return {AgentSocket} */ find(browserSocket) { - const token = browserSocket.request.user.token; + const {_id} = browserSocket.request.user; - const sockets = this.sockets.get(token); + const sockets = this.sockets.get(_id); return _.find(sockets, (sock) => _.includes(sock.demo.browserSockets, browserSocket)); } @@ -228,19 +228,26 @@ module.exports.factory = function(settings, mongo, AgentSocket) { * Link agent with browsers by account. * * @param {Socket} sock + * @param {Array.} accounts * @param {Array.} tokens * @param {boolean} demoEnabled * * @private */ - onConnect(sock, tokens, demoEnabled) { - const agentSocket = new AgentSocket(sock, tokens, demoEnabled); + onConnect(sock, accounts, tokens, demoEnabled) { + const agentSocket = new AgentSocket(sock, accounts, tokens, demoEnabled); + + _.forEach(accounts, (account) => { + this._agentSockets.add(account, agentSocket); + + this._browsersHnd.agentStats(account); + }); sock.on('disconnect', () => { - _.forEach(tokens, (token) => { - _.pull(this._agentSockets.get(token), agentSocket); + _.forEach(accounts, (account) => { + _.pull(this._agentSockets.get(account), agentSocket); - this._browsersHnd.agentStats(token); + this._browsersHnd.agentStats(account); }); }); @@ -258,8 +265,8 @@ module.exports.factory = function(settings, mongo, AgentSocket) { if (agentSocket.cluster !== cluster) { agentSocket.cluster = cluster; - _.forEach(tokens, (token) => { - this._browsersHnd.agentStats(token); + _.forEach(accounts, (account) => { + this._browsersHnd.agentStats(account); }); } else { @@ -268,8 +275,8 @@ module.exports.factory = function(settings, mongo, AgentSocket) { if (changed) { cluster.update(top); - _.forEach(tokens, (token) => { - this._browsersHnd.clusterChanged(token, cluster); + _.forEach(accounts, (account) => { + this._browsersHnd.clusterChanged(account, cluster); }); } } @@ -282,16 +289,17 @@ module.exports.factory = function(settings, mongo, AgentSocket) { agentSocket.cluster = null; - _.forEach(tokens, (token) => { - this._browsersHnd.agentStats(token); + _.forEach(accounts, (account) => { + this._browsersHnd.agentStats(account); }); }); - _.forEach(tokens, (token) => { - this._agentSockets.add(token, agentSocket); + return agentSocket; + } - this._browsersHnd.agentStats(token); - }); + getAccounts(tokens) { + return mongo.Account.find({token: {$in: tokens}}, '_id token').lean().exec() + .then((accounts) => ({accounts, activeTokens: _.uniq(_.map(accounts, 'token'))})); } /** @@ -301,17 +309,27 @@ module.exports.factory = function(settings, mongo, AgentSocket) { attach(srv, browsersHnd) { this._browsersHnd = browsersHnd; - if (this.io) - throw 'Agent server already started!'; - this._collectSupportedAgents() .then((supportedAgents) => { this.currentAgent = _.get(supportedAgents, 'current'); + if (this.io) + throw 'Agent server already started!'; + this.io = socketio(srv, {path: '/agents'}); this.io.on('connection', (sock) => { + const sockId = sock.id; + + console.log('Connected agent with socketId: ', sockId); + + sock.on('disconnect', (reason) => { + console.log(`Agent disconnected with [socketId=${sockId}, reason=${reason}]`); + }); + sock.on('agent:auth', ({ver, bt, tokens, disableDemo} = {}, cb) => { + console.log(`Received authentication request [socketId=${sockId}, tokens=${tokens}, ver=${ver}].`); + if (_.isEmpty(tokens)) return cb('Tokens not set. Please reload agent archive or check settings'); @@ -322,32 +340,33 @@ module.exports.factory = function(settings, mongo, AgentSocket) { return cb('You are using an older version of the agent. Please reload agent'); } - return mongo.Account.find({token: {$in: tokens}}, '_id token').lean().exec() - .then((accounts) => { - const activeTokens = _.uniq(_.map(accounts, 'token')); - + return this.getAccounts(tokens) + .then(({accounts, activeTokens}) => { if (_.isEmpty(activeTokens)) return cb(`Failed to authenticate with token(s): ${tokens.join(',')}. Please reload agent archive or check settings`); cb(null, activeTokens); - return this.onConnect(sock, activeTokens, disableDemo); + return this.onConnect(sock, accounts, activeTokens, disableDemo); }) // TODO IGNITE-1379 send error to web master. .catch(() => cb(`Invalid token(s): ${tokens.join(',')}. Please reload agent archive or check settings`)); }); }); + }) + .catch(() => { + console.log('Failed to collect supported agents'); }); } - agent(token, demo, clusterId) { + agent(account, demo, clusterId) { if (!this.io) return Promise.reject(new Error('Agent server not started yet!')); - const socks = this._agentSockets.get(token); + const socks = this._agentSockets.get(account); if (_.isEmpty(socks)) - return Promise.reject(new Error('Failed to find connected agent for this token')); + return Promise.reject(new Error('Failed to find connected agent for this account')); if (demo || _.isNil(clusterId)) return Promise.resolve(_.head(socks)); @@ -360,11 +379,11 @@ module.exports.factory = function(settings, mongo, AgentSocket) { return Promise.resolve(sock); } - agents(token) { + agents(account) { if (!this.io) return Promise.reject(new Error('Agent server not started yet!')); - const socks = this._agentSockets.get(token); + const socks = this._agentSockets.get(account); if (_.isEmpty(socks)) return Promise.reject(new Error('Failed to find connected agent for this token')); @@ -372,61 +391,18 @@ module.exports.factory = function(settings, mongo, AgentSocket) { return Promise.resolve(socks); } - tryStopDemo(browserSocket) { - const agentSock = this._agentSockets.find(browserSocket); - } - - /** - * @param {ObjectId} token - * @param {Socket} browserSock - * @returns {int} Connected agent count. - */ - onBrowserConnect(token, browserSock) { - this.emitAgentsCount(token); - - // If connect from browser with enabled demo. - const demo = browserSock.request._query.IgniteDemoMode === 'true'; - - // Agents where possible to run demo. - const agentSockets = _.filter(this._agentSockets[token], 'demo.enabled'); - - if (demo && _.size(agentSockets)) { - const agentSocket = _.find(agentSockets, (agent) => _.includes(agent.demo.tokens, token)); - - if (agentSocket) - agentSocket.attachToDemoCluster(browserSock); - else - _.head(agentSockets).runDemoCluster(token, [browserSock]); - } - } - - /** - * @param {Socket} browserSock. - */ - onBrowserDisconnect(browserSock) { - const token = browserSock.client.request.user.token; - - this._browserSockets.pull(token, browserSock); - - // If connect from browser with enabled demo. - if (browserSock.request._query.IgniteDemoMode === 'true') - this._agentSockets.find(token, (agent) => _.includes(agent.demo.browserSockets, browserSock)); - - // TODO If latest browser with demo need stop demo cluster on agent. - } - /** * Try stop agent for token if not used by others. * - * @param {String} token + * @param {mongo.Account} account */ - onTokenReset(token) { + onTokenReset(account) { if (_.isNil(this.io)) return; - const sockets = this._agentSockets[token]; + const agentSockets = this._agentSockets.get(account); - _.forEach(sockets, (socket) => socket._sendToAgent('agent:reset:token', token)); + _.forEach(agentSockets, (sock) => sock.resetToken(account.token)); } } diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js index d0cd1128406d4..a18b4673d851b 100644 --- a/modules/web-console/backend/app/browsersHandler.js +++ b/modules/web-console/backend/app/browsersHandler.js @@ -38,14 +38,14 @@ module.exports = { * @param {Socket} sock */ add(sock) { - const token = sock.request.user.token; + const token = sock.request.user._id.toString(); if (this.sockets.has(token)) this.sockets.get(token).push(sock); else this.sockets.set(token, [sock]); - return this.sockets.get(token); + return this.sockets.get(sock.request.user); } /** @@ -61,11 +61,13 @@ module.exports = { return sockets; } - get(token) { - if (this.sockets.has(token)) - return this.sockets.get(token); + get(account) { + let sockets = this.sockets.get(account._id.toString()); - return []; + if (_.isEmpty(sockets)) + this.sockets.set(account._id.toString(), sockets = []); + + return sockets; } demo(token) { @@ -103,11 +105,11 @@ module.exports = { } /** - * @param {String} token + * @param {String} account * @param {Array.} [socks] */ - agentStats(token, socks = this._browserSockets.get(token)) { - return this._agentHnd.agents(token) + agentStats(account, socks = this._browserSockets.get(account)) { + return this._agentHnd.agents(account) .then((agentSocks) => { const stat = _.reduce(agentSocks, (acc, agentSock) => { acc.count += 1; @@ -127,8 +129,8 @@ module.exports = { .then((stat) => _.forEach(socks, (sock) => sock.emit('agents:stat', stat))); } - clusterChanged(token, cluster) { - const socks = this._browserSockets.get(token); + clusterChanged(account, cluster) { + const socks = this._browserSockets.get(account); _.forEach(socks, (sock) => sock.emit('cluster:changed', cluster)); } @@ -153,10 +155,10 @@ module.exports = { } } - executeOnAgent(token, demo, event, ...args) { + executeOnAgent(account, demo, event, ...args) { const cb = _.last(args); - return this._agentHnd.agent(token, demo) + return this._agentHnd.agent(account, demo) .then((agentSock) => agentSock.emitEvent(event, ..._.dropRight(args))) .then((res) => cb(null, res)) .catch((err) => cb(this.errorTransformer(err))); @@ -164,21 +166,21 @@ module.exports = { agentListeners(sock) { const demo = sock.request._query.IgniteDemoMode === 'true'; - const token = () => sock.request.user.token; + const account = () => sock.request.user; // Return available drivers to browser. sock.on('schemaImport:drivers', (...args) => { - this.executeOnAgent(token(), demo, 'schemaImport:drivers', ...args); + this.executeOnAgent(account(), demo, 'schemaImport:drivers', ...args); }); // Return schemas from database to browser. sock.on('schemaImport:schemas', (...args) => { - this.executeOnAgent(token(), demo, 'schemaImport:schemas', ...args); + this.executeOnAgent(account(), demo, 'schemaImport:schemas', ...args); }); // Return tables from database to browser. sock.on('schemaImport:metadata', (...args) => { - this.executeOnAgent(token(), demo, 'schemaImport:metadata', ...args); + this.executeOnAgent(account(), demo, 'schemaImport:metadata', ...args); }); } @@ -218,9 +220,7 @@ module.exports = { return cb('Invalid format of message: "node:rest"'); } - const token = sock.request.user.token; - - const agent = this._agentHnd.agent(token, demo, clusterId); + const agent = this._agentHnd.agent(sock.request.user, demo, clusterId); this.executeOnNode(agent, demo, credentials, params) .then((data) => cb(null, data)) @@ -259,8 +259,6 @@ module.exports = { return cb('Invalid format of message: "node:visor"'); } - const token = sock.request.user.token; - const {taskId, nids, args = []} = params; const desc = this._visorTasks.get(taskId); @@ -277,7 +275,7 @@ module.exports = { _.forEach(_.concat(desc.argCls, args), (param, idx) => { exeParams[`p${idx + 3}`] = param; }); - const agent = this._agentHnd.agent(token, demo, clusterId); + const agent = this._agentHnd.agent(sock.request.user, demo, clusterId); this.executeOnNode(agent, demo, credentials, exeParams) .then((data) => { @@ -317,18 +315,13 @@ module.exports = { // Handle browser disconnect event. sock.on('disconnect', () => { this._browserSockets.remove(sock); - - const demo = sock.request._query.IgniteDemoMode === 'true'; - - // Stop demo if latest demo tab for this token. - demo && agentHnd.tryStopDemo(sock); }); this.agentListeners(sock); this.nodeListeners(sock); this.pushInitialData(sock); - this.agentStats(sock.request.user.token, [sock]); + this.agentStats(sock.request.user, [sock]); this.emitNotification(sock); }); }); diff --git a/modules/web-console/backend/app/configure.js b/modules/web-console/backend/app/configure.js index b7bdb49097d33..a0e5190c8b2d9 100644 --- a/modules/web-console/backend/app/configure.js +++ b/modules/web-console/backend/app/configure.js @@ -47,8 +47,6 @@ module.exports.factory = function(settings, mongo, apis) { _.forEach(apis, (api) => app.use(api)); - app.use(cookieParser(settings.sessionSecret)); - app.use(bodyParser.json({limit: '50mb'})); app.use(bodyParser.urlencoded({limit: '50mb', extended: true})); @@ -67,18 +65,26 @@ module.exports.factory = function(settings, mongo, apis) { app.use(passport.initialize()); app.use(passport.session()); - passport.serializeUser(mongo.Account.serializeUser()); - passport.deserializeUser(mongo.Account.deserializeUser()); + passport.serializeUser((user, done) => done(null, user._id)); + + passport.deserializeUser((id, done) => { + if (mongo.ObjectId.isValid(id)) + return mongo.Account.findById(id, done); + + // Invalidates the existing login session. + done(null, false); + }); passport.use(mongo.Account.createStrategy()); }, socketio: (io) => { - const _onAuthorizeSuccess = (data, accept) => { - accept(null, true); - }; + const _onAuthorizeSuccess = (data, accept) => accept(); const _onAuthorizeFail = (data, message, error, accept) => { - accept(null, false); + if (error) + accept(new Error(message)); + + return accept(new Error(message)); }; io.use(passportSocketIo.authorize({ diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js index d206107843908..104b66d690196 100644 --- a/modules/web-console/backend/app/settings.js +++ b/modules/web-console/backend/app/settings.js @@ -61,7 +61,8 @@ module.exports = { server: { host: nconf.get('server:host') || dfltHost, port: _normalizePort(nconf.get('server:port') || dfltPort), - SSLOptions: nconf.get('server:ssl') && { + // eslint-disable-next-line eqeqeq + SSLOptions: nconf.get('server:ssl') == 'true' && { enable301Redirects: true, trustXFPHeader: true, key: fs.readFileSync(nconf.get('server:key')), From ae4c89eb92199723cbde869062e35b7b160d764b Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Mon, 8 Oct 2018 10:34:46 +0700 Subject: [PATCH 117/403] IGNITE-9809 Web Console: Minor fixes. (cherry picked from commit 1ec9ee43884ccafd0ebbb2cbcfbd596ce79dbfd6) --- .../web-console/frontend/app/app.config.js | 3 +- .../app/components/bs-select-menu/style.scss | 2 +- .../components/cluster-selector/template.pug | 2 +- .../list-editable-cols/row.directive.js | 5 +- .../page-queries/notebook.service.js | 2 +- .../web-console-header/component.js | 4 + .../web-console-header/template.pug | 5 +- .../app/core/activities/Activities.data.js | 5 +- modules/web-console/frontend/app/data/i18n.js | 256 +++++++++++++++++- .../frontend/app/filters/duration.filter.js | 6 +- .../app/modules/agent/AgentManager.service.js | 15 +- .../frontend/app/modules/user/User.service.js | 7 + .../frontend/app/modules/user/user.module.js | 21 +- .../app/primitives/tooltip/index.scss | 2 +- .../app/services/FormUtils.service.js | 2 +- .../frontend/app/services/exceptionHandler.js | 4 + modules/web-console/frontend/package.json | 5 +- .../stylesheets/_bootstrap-variables.scss | 2 +- .../frontend/public/stylesheets/style.scss | 1 - 19 files changed, 316 insertions(+), 33 deletions(-) diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js index e2bc057fc4ece..88bffee7ee746 100644 --- a/modules/web-console/frontend/app/app.config.js +++ b/modules/web-console/frontend/app/app.config.js @@ -15,8 +15,8 @@ * limitations under the License. */ -import _ from 'lodash'; import angular from 'angular'; + import negate from 'lodash/negate'; import isNil from 'lodash/isNil'; import isEmpty from 'lodash/isEmpty'; @@ -133,4 +133,3 @@ igniteConsoleCfg.directive('uiGridSelection', function() { } }; }); - diff --git a/modules/web-console/frontend/app/components/bs-select-menu/style.scss b/modules/web-console/frontend/app/components/bs-select-menu/style.scss index ac3991b4957dc..ac3f7aa39f988 100644 --- a/modules/web-console/frontend/app/components/bs-select-menu/style.scss +++ b/modules/web-console/frontend/app/components/bs-select-menu/style.scss @@ -99,4 +99,4 @@ bottom: 0; left: 0; z-index: 1999; -} \ No newline at end of file +} diff --git a/modules/web-console/frontend/app/components/cluster-selector/template.pug b/modules/web-console/frontend/app/components/cluster-selector/template.pug index f1e80db67f2c4..39c1af29abada 100644 --- a/modules/web-console/frontend/app/components/cluster-selector/template.pug +++ b/modules/web-console/frontend/app/components/cluster-selector/template.pug @@ -45,7 +45,7 @@ span(data-ng-if='!$ctrl.isDemo && $ctrl.clusters.length > 1') aria-expanded='false' ) span(ng-if='!$ctrl.cluster') No clusters available - + span(ng-if='$ctrl.cluster') cluster-security-icon(secured='$ctrl.cluster.secured') | {{ $ctrl.cluster.name }} diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js index 2753263ef03b3..8f3833119d8ce 100644 --- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js +++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js @@ -15,6 +15,8 @@ * limitations under the License. */ +import {nonEmpty} from 'app/utils/lodashMixins'; + import {ListEditableColsController} from './cols.directive'; /** @returns {ng.IDirective} */ @@ -35,7 +37,8 @@ export default function() { el.addClass(ctrl.rowClass); ctrl.colDefs.forEach(({ cellClass }, index) => { - _.forEach((Array.isArray(cellClass) ? cellClass : [cellClass]), (item) => children[index].classList.add(item)); + if (nonEmpty(cellClass)) + children[index].classList.add(...(Array.isArray(cellClass) ? cellClass : cellClass.split(' '))); }); } }; diff --git a/modules/web-console/frontend/app/components/page-queries/notebook.service.js b/modules/web-console/frontend/app/components/page-queries/notebook.service.js index dbfd6a6febf77..2a60cdcd74515 100644 --- a/modules/web-console/frontend/app/components/page-queries/notebook.service.js +++ b/modules/web-console/frontend/app/components/page-queries/notebook.service.js @@ -70,7 +70,7 @@ export default class Notebook { return this.confirmModal.confirm(`Are you sure you want to remove notebook: "${notebook.name}"?`) .then(() => this.NotebookData.findIndex(notebook)) .then((idx) => { - this.NotebookData.remove(notebook) + return this.NotebookData.remove(notebook) .then(() => { if (this.$state.includes('base.sql.tabs.notebook') && this.$state.params.noteId === notebook._id) return this._openNotebook(idx); diff --git a/modules/web-console/frontend/app/components/web-console-header/component.js b/modules/web-console/frontend/app/components/web-console-header/component.js index a0a8230c17863..02b30801e613b 100644 --- a/modules/web-console/frontend/app/components/web-console-header/component.js +++ b/modules/web-console/frontend/app/components/web-console-header/component.js @@ -36,6 +36,10 @@ export default { !this.constructor.connectedClustersUnvisibleStates.some((state) => this.$state.includes(state)); } + isAuthorized() { + return !!this.$rootScope.user; + } + $onInit() { this.setConnectedClustersVisible(); diff --git a/modules/web-console/frontend/app/components/web-console-header/template.pug b/modules/web-console/frontend/app/components/web-console-header/template.pug index 801b742e949d1..1d25b13289010 100644 --- a/modules/web-console/frontend/app/components/web-console-header/template.pug +++ b/modules/web-console/frontend/app/components/web-console-header/template.pug @@ -26,7 +26,10 @@ .wch-content connected-clusters(ng-if='$ctrl.$rootScope.user && !$ctrl.$rootScope.IgniteDemoMode && $ctrl.isConnectedClustersVisible && !$root.user.becomeUsed') - a(ui-sref='landing') + a(ui-sref='default-state' ng-if='$ctrl.isAuthorized()').wch-logo-anchor + img.wch-logo(ng-src='{{::$ctrl.branding.headerLogo}}') + + a(ui-sref='landing' ng-if='!$ctrl.isAuthorized()').wch-logo-anchor img.wch-logo(ng-src='{{::$ctrl.branding.headerLogo}}') .wch-slot.wch-slot-left(ng-transclude='slotLeft') diff --git a/modules/web-console/frontend/app/core/activities/Activities.data.js b/modules/web-console/frontend/app/core/activities/Activities.data.js index 35b44e63bf55e..8786816422e1e 100644 --- a/modules/web-console/frontend/app/core/activities/Activities.data.js +++ b/modules/web-console/frontend/app/core/activities/Activities.data.js @@ -35,6 +35,9 @@ export default class ActivitiesData { action = action || this.$state.$current.url.source || ''; group = group || (action.match(/^\/([^/]+)/) || [])[1]; - return this.$http.post('/api/v1/activities/page', { group, action }); + return this.$http.post('/api/v1/activities/page', { group, action }) + .catch(() => { + // No-op. + }); } } diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js index e21b8c5c338c8..e6c1294dfce70 100644 --- a/modules/web-console/frontend/app/data/i18n.js +++ b/modules/web-console/frontend/app/data/i18n.js @@ -43,5 +43,259 @@ export default { '/queries/notebook/': 'Query notebook', '/settings/profile': 'User profile', '/settings/admin': 'Admin panel', - '/logout': 'Logout' + '/logout': 'Logout', + + // app/components/page-signin/template.pug + 'app.components.page-signin.m1': 'Sign In', + 'app.components.page-signin.m2': 'Email:', + 'app.components.page-signin.m3': 'Input email', + 'app.components.page-signin.m4': 'Password:', + 'app.components.page-signin.m5': 'Input password', + 'app.components.page-signin.m6': 'Forgot password?', + 'app.components.page-signin.m7': 'Sign In', + 'app.components.page-signin.m8': 'Don\'t have an account? #[a(ui-sref=\'signup\') Get started]', + // app/components/page-signin/run.js + 'app.components.page-signin.m9': 'Sign In', + + + // app/components/page-queries/template.tpl.pug + 'app.components.page-queries.m1': 'Show data in tabular form', + 'app.components.page-queries.m2': 'Show bar chart
By default first column - X values, second column - Y values
In case of one column it will be treated as Y values', + 'app.components.page-queries.m3': 'Show pie chart
By default first column - pie labels, second column - pie values
In case of one column it will be treated as pie values', + 'app.components.page-queries.m4': 'Show line chart
By default first column - X values, second column - Y values
In case of one column it will be treated as Y values', + 'app.components.page-queries.m5': 'Show area chart
By default first column - X values, second column - Y values
In case of one column it will be treated as Y values', + 'app.components.page-queries.m6': 'Click to show chart settings dialog', + 'app.components.page-queries.m7': 'Chart settings', + 'app.components.page-queries.m8': 'Show', + 'app.components.page-queries.m9': 'min', + 'app.components.page-queries.m10': 'Duration: #[b {{paragraph.duration | duration}}]', + 'app.components.page-queries.m11': 'NodeID8: #[b {{paragraph.resNodeId | id8}}]', + 'app.components.page-queries.m12': 'Rename notebook', + 'app.components.page-queries.m13': 'Remove notebook', + 'app.components.page-queries.m14': 'Save notebook name', + 'app.components.page-queries.m15': 'Scroll to query', + 'app.components.page-queries.m16': 'Add query', + 'app.components.page-queries.m17': 'Add scan', + 'app.components.page-queries.m18': 'Failed to load notebook', + 'app.components.page-queries.m19': 'Notebook not accessible any more. Leave notebooks or open another notebook.', + 'app.components.page-queries.m20': 'Leave notebooks', + 'app.components.page-queries.m21': 'Rename query', + 'app.components.page-queries.m22': 'Rename query', + 'app.components.page-queries.m23': 'Remove query', + 'app.components.page-queries.m24': 'Save query name', + 'app.components.page-queries.m25': 'Configure periodical execution of last successfully executed query', + 'app.components.page-queries.m26': 'Refresh rate:', + 'app.components.page-queries.m27': 'Max number of rows to show in query result as one page', + 'app.components.page-queries.m28': 'Page size:', + 'app.components.page-queries.m29': 'Limit query max results to specified number of pages', + 'app.components.page-queries.m30': 'Max pages:', + 'app.components.page-queries.m31': 'Non-collocated joins is a special mode that allow to join data across cluster without collocation.
Nested joins are not supported for now.
NOTE: In some cases it may consume more heap memory or may take a long time than collocated joins.', + 'app.components.page-queries.m32': 'Allow non-collocated joins', + 'app.components.page-queries.m33': 'Enforce join order of tables in the query.
If set, then query optimizer will not reorder tables within join.
NOTE: It is not recommended to enable this property unless you have verified that indexes are not selected in optimal order.', + 'app.components.page-queries.m34': 'Enforce join order', + 'app.components.page-queries.m35': 'By default Ignite attempts to fetch the whole query result set to memory and send it to the client.
For small and medium result sets this provides optimal performance and minimize duration of internal database locks, thus increasing concurrency.
If result set is too big to fit in available memory this could lead to excessive GC pauses and even OutOfMemoryError.
Use this flag as a hint for Ignite to fetch result set lazily, thus minimizing memory consumption at the cost of moderate performance hit.', + 'app.components.page-queries.m36': 'Lazy result set', + 'app.components.page-queries.m37': 'Execute', + 'app.components.page-queries.m38': 'Execute on selected node', + 'app.components.page-queries.m39': '{{queryTooltip(paragraph, "explain query")}}', + 'app.components.page-queries.m40': 'Explain', + 'app.components.page-queries.m41': 'Page: #[b {{paragraph.page}}]', + 'app.components.page-queries.m42': 'Results so far: #[b {{paragraph.rows.length + paragraph.total}}]', + 'app.components.page-queries.m43': 'Duration: #[b {{paragraph.duration | duration}}]', + 'app.components.page-queries.m44': 'NodeID8: #[b {{paragraph.resNodeId | id8}}]', + 'app.components.page-queries.m45': '{{ queryTooltip(paragraph, "export query results") }}', + 'app.components.page-queries.m46': 'Export', + 'app.components.page-queries.m47': 'Export', + 'app.components.page-queries.m48': 'Export all', + 'app.components.page-queries.m49': 'Copy current result page to clipboard', + 'app.components.page-queries.m50': 'Copy to clipboard', + 'app.components.page-queries.m51': 'Page: #[b {{paragraph.pa', + 'app.components.page-queries.m52': 'Results so far: #[b {{paragraph.rows.length + paragraph.total}}]', + 'app.components.page-queries.m53': 'Duration: #[b {{paragraph.duration | duration}}]', + 'app.components.page-queries.m54': 'NodeID8: #[b {{paragraph.resNodeId | id8}}]', + 'app.components.page-queries.m55': 'Export', + 'app.components.page-queries.m56': 'Export', + 'app.components.page-queries.m57': 'Export all', + 'app.components.page-queries.m58': 'Copy current result page to clipboard', + 'app.components.page-queries.m59': 'Copy to clipboard', + 'app.components.page-queries.m60': 'Cannot display chart. Please configure axis using #[b Chart settings]', + 'app.components.page-queries.m61': 'Cannot display chart. Result set must contain Java build-in type columns. Please change query and execute it again.', + 'app.components.page-queries.m62': 'Pie chart does not support \'TIME_LINE\' column for X-axis. Please use another column for X-axis or switch to another chart.', + 'app.components.page-queries.m63': 'Charts do not support #[b Explain] and #[b Scan] query', + 'app.components.page-queries.m64': 'Cache:', + 'app.components.page-queries.m65': 'Choose cache', + 'app.components.page-queries.m66': 'Filter:', + 'app.components.page-queries.m67': 'Enter filter', + 'app.components.page-queries.m68': 'Select this checkbox for case sensitive search', + 'app.components.page-queries.m69': 'Max number of rows to show in query result as one page', + 'app.components.page-queries.m70': 'Page size:', + 'app.components.page-queries.m71': 'Scan', + 'app.components.page-queries.m72': 'Scan on selected node', + 'app.components.page-queries.m73': 'Error: {{paragraph.error.message}}', + 'app.components.page-queries.m74': 'Result set is empty. Duration: #[b {{paragraph.duration | duration}}]', + 'app.components.page-queries.m75': 'Showing results for scan of #[b {{ paragraph.queryArgs.cacheName | defaultName }}]', + 'app.components.page-queries.m76': '  with filter: #[b {{ paragraph.queryArgs.filter }}]', + 'app.components.page-queries.m77': '  on node: #[b {{ paragraph.queryArgs.localNid | limitTo:8 }}]', + 'app.components.page-queries.m78': 'Next', + 'app.components.page-queries.m79': 'Caches:', + 'app.components.page-queries.m80': 'Click to show cache types metadata dialog', + 'app.components.page-queries.m81': 'Filter caches...', + 'app.components.page-queries.m82': 'Use selected cache as default schema name.
This will allow to execute query on specified cache without specify schema name.
NOTE: In future version of Ignite this feature will be removed.', + 'app.components.page-queries.m83': 'Use selected cache as default schema name', + 'app.components.page-queries.m84': 'Wrong caches filter', + 'app.components.page-queries.m85': 'No caches', + 'app.components.page-queries.m86': 'Error: {{paragraph.error.message}}', + 'app.components.page-queries.m87': 'Show more', + 'app.components.page-queries.m88': 'Show query', + 'app.components.page-queries.m89': 'Next', + 'app.components.page-queries.m90': 'Queries', + 'app.components.page-queries.m91': 'With query notebook you can', + 'app.components.page-queries.m92': 'Create any number of queries', + 'app.components.page-queries.m93': 'Execute and explain SQL queries', + 'app.components.page-queries.m94': 'Execute scan queries', + 'app.components.page-queries.m95': 'View data in tabular form and as charts', + 'app.components.page-queries.m96': 'Examples:', + // app/components/page-queries/Notebook.service.js + 'app.components.page-queries.m97': 'Are you sure you want to remove notebook: "${notebook.name}"?', + // app/components/page-queries/Notebook.data.js + 'app.components.page-queries.m98': 'SQL demo', + 'app.components.page-queries.m99': 'Query with refresh rate', + 'app.components.page-queries.m100': 'Simple query', + 'app.components.page-queries.m101': 'Query with aggregates', + 'app.components.page-queries.m102': 'Failed to load notebook.', + 'app.components.page-queries.m103': 'Removing "${notebook.name}" notebook is not supported.', + // app/components/page-queries/index.js + 'app.components.page-queries.m104': 'Query notebook', + 'app.components.page-queries.m105': 'SQL demo', + // app/components/page-queries/controller.js + 'app.components.page-queries.m106': 'Internal cluster error', + 'app.components.page-queries.m107': 'Unlimited', + 'app.components.page-queries.m108': 'Demo grid is starting. Please wait...', + 'app.components.page-queries.m109': 'Loading query notebook screen...', + 'app.components.page-queries.m110': 'seconds', + 'app.components.page-queries.m111': 's', + 'app.components.page-queries.m112': 'minutes', + 'app.components.page-queries.m113': 'm', + 'app.components.page-queries.m114': 'hours', + 'app.components.page-queries.m115': 'h', + 'app.components.page-queries.m116': 'Leave Queries', + 'app.components.page-queries.m117': 'Query ${sz === 0 ? "" : sz}', + 'app.components.page-queries.m118': 'Scan ${sz === 0 ? "" : sz}', + 'app.components.page-queries.m119': 'Are you sure you want to remove query: "${paragraph.name}"?', + 'app.components.page-queries.m120': 'Waiting for server response', + 'app.components.page-queries.m121': 'Input text to ${action}', + 'app.components.page-queries.m122': 'Waiting for server response', + 'app.components.page-queries.m123': 'Select cache to export scan results', + 'app.components.page-queries.m124': 'SCAN query', + 'app.components.page-queries.m125': 'SCAN query for cache: ${maskCacheName(paragraph.queryArgs.cacheName, true)}', + 'app.components.page-queries.m126': 'SCAN query for cache: ${maskCacheName(paragraph.queryArgs.cacheName, true)} with filter: ${filter}', + 'app.components.page-queries.m127': 'Explain query', + 'app.components.page-queries.m128': 'SQL query', + 'app.components.page-queries.m129': 'Duration: ${$filter(\'duration\')(paragraph.duration)}.', + 'app.components.page-queries.m130': 'Node ID8: ${_.id8(paragraph.resNodeId)}', + 'app.components.page-queries.m131': 'Error details', + + // app/components/page-queries/services/queries-navbar.js + 'app.components.page-queries.services.queries-navbar.m1': 'Queries', + 'app.components.page-queries.services.queries-navbar.m2': 'Create new notebook', + + // app/components/page-queries/services/create-query-dialog/template.pug + 'app.components.page-queries.services.create-query-dialog.m1': 'New query notebook', + 'app.components.page-queries.services.create-query-dialog.m2': 'Name: ', + 'app.components.page-queries.services.create-query-dialog.m3': 'Cancel', + 'app.components.page-queries.services.create-query-dialog.m4': 'Create', + + // app/components/page-profile/controller.js + 'app.components.page-profile.m1': 'Are you sure you want to change security token?', + 'app.components.page-profile.m2': 'Profile saved.', + 'app.components.page-profile.m3': 'Failed to save profile: ', + // app/components/page-profile/index.js + 'app.components.page-profile.m4': 'User profile', + // app/components/page-profile/template.pug + 'app.components.page-profile.m5': 'User profile', + 'app.components.page-profile.m6': 'First name:', + 'app.components.page-profile.m7': 'Input first name', + 'app.components.page-profile.m8': 'Last name:', + 'app.components.page-profile.m9': 'Input last name', + 'app.components.page-profile.m10': 'Email:', + 'app.components.page-profile.m11': 'Input email', + 'app.components.page-profile.m12': 'Phone:', + 'app.components.page-profile.m13': 'Input phone (ex.: +15417543010)', + 'app.components.page-profile.m14': 'Country:', + 'app.components.page-profile.m15': 'Choose your country', + 'app.components.page-profile.m16': 'Company:', + 'app.components.page-profile.m17': 'Input company name', + 'app.components.page-profile.m18': 'Cancel security token changing...', + 'app.components.page-profile.m19': 'Show security token...', + 'app.components.page-profile.m20': 'Security token:', + 'app.components.page-profile.m21': 'No security token. Regenerate please.', + 'app.components.page-profile.m22': 'Generate random security token', + 'app.components.page-profile.m23': 'Copy security token to clipboard', + 'app.components.page-profile.m24': 'The security token is used for authorization of web agent', + 'app.components.page-profile.m25': 'Cancel password changing...', + 'app.components.page-profile.m26': 'Change password...', + 'app.components.page-profile.m27': 'New password:', + 'app.components.page-profile.m28': 'New password', + 'app.components.page-profile.m29': 'Confirm password:', + 'app.components.page-profile.m30': 'Confirm new password', + 'app.components.page-profile.m31': 'Cancel', + 'app.components.page-profile.m32': 'Save Changes', + + // app/modules/navbar/userbar.directive.js + 'app/modules/navbar/userbar.m1': 'Profile', + 'app/modules/navbar/userbar.m2': 'Getting started', + 'app/modules/navbar/userbar.m3': 'Admin panel', + 'app/modules/navbar/userbar.m4': 'Log out', + + // app/components/page-forgot-password/run.js + 'app.components.page-forgot-password.m1': 'Forgot Password', + // app/components/page-forgot-password/template.pug + 'app.components.page-forgot-password.m2': 'Forgot password?', + 'app.components.page-forgot-password.m3': 'Enter the email address for your account & we\'ll email you a link to reset your password.', + 'app.components.page-forgot-password.m4': 'Email:', + 'app.components.page-forgot-password.m5': 'Input email', + 'app.components.page-forgot-password.m6': 'Back to sign in', + 'app.components.page-forgot-password.m7': 'Send it to me', + + // app/components/page-landing/template.pug + 'app.components.page-landing.m1': 'Sign In', + 'app.components.page-landing.m2': 'Web Console', + 'app.components.page-landing.m3': 'An Interactive Configuration Wizard and Management Tool for Apache™ Ignite®', + 'app.components.page-landing.m4': 'It provides an interactive configuration wizard which helps you create and download configuration files and code snippets for your Apache Ignite projects. Additionally, the tool allows you to automatically load SQL metadata from any RDBMS, run SQL queries on your in-memory cache, and view execution plans, in-memory schema, and streaming charts.', + 'app.components.page-landing.m5': 'Sign Up', + 'app.components.page-landing.m6': 'The Web Console allows you to:', + 'app.components.page-landing.m7': 'Configure Apache Ignite clusters and caches', + 'app.components.page-landing.m8': 'The Web Console configuration wizard takes you through a step-by-step process that helps you define all the required configuration parameters. The system then generates a ready-to-use project with all the required config files.', + 'app.components.page-landing.m9': 'Run free-form SQL queries on #[br] Apache Ignite caches', + 'app.components.page-landing.m10': 'By connecting the Web Console to your Apache Ignite cluster, you can execute SQL queries on your in-memory cache. You can also view the execution plan, in-memory schema, and streaming charts for your cluster.', + 'app.components.page-landing.m11': 'Import database schemas from #[br] virtually any RDBMS', + 'app.components.page-landing.m12': 'To speed the creation of your configuration files, the Web Console allows you to automatically import the database schema from virtually any RDBMS including Oracle, SAP, MySQL, PostgreSQL, and many more.', + 'app.components.page-landing.m13': 'Manage the Web Console users', + 'app.components.page-landing.m14': 'The Web Console allows you to have accounts with different roles.', + 'app.components.page-landing.m15': 'Get Started', + + // app/components/page-signup/template.pug + 'app.components.page-signup.m1': 'Don\'t Have An Account?', + 'app.components.page-signup.m2': 'Email:', + 'app.components.page-signup.m3': 'Input email', + 'app.components.page-signup.m4': 'Password:', + 'app.components.page-signup.m5': 'Input password', + 'app.components.page-signup.m6': 'Confirm:', + 'app.components.page-signup.m7': 'Confirm password', + 'app.components.page-signup.m8': 'First name:', + 'app.components.page-signup.m9': 'Input first name', + 'app.components.page-signup.m10': 'Last name:', + 'app.components.page-signup.m11': 'Input last name', + 'app.components.page-signup.m12': 'Phone:', + 'app.components.page-signup.m13': 'Input phone (ex.: +15417543010)', + 'app.components.page-signup.m14': 'Country:', + 'app.components.page-signup.m15': 'Choose your country', + 'app.components.page-signup.m16': 'Company:', + 'app.components.page-signup.m17': 'Input company name', + 'app.components.page-signup.m18': 'Sign Up', + 'app.components.page-signup.m19': 'Already have an account? #[a(ui-sref=\'signin\') Sign in here]', + + // app/components/password-visibility/toggle-button.component.js + 'app.components.password-visibility.m1': 'Hide password', + 'app.components.password-visibility.m2': 'Show password' }; diff --git a/modules/web-console/frontend/app/filters/duration.filter.js b/modules/web-console/frontend/app/filters/duration.filter.js index 810cc4d44678f..0ce42265627de 100644 --- a/modules/web-console/frontend/app/filters/duration.filter.js +++ b/modules/web-console/frontend/app/filters/duration.filter.js @@ -18,11 +18,15 @@ export default () => { /** * @param {number} t Time in ms. + * @param {string} dflt Default value. */ - const filter = (t) => { + const filter = (t, dflt = '0') => { if (t === 9223372036854775807) return 'Infinite'; + if (t <= 0) + return dflt; + const a = (i, suffix) => i && i !== '00' ? i + suffix + ' ' : ''; const cd = 24 * 60 * 60 * 1000; diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index aa949a2d7a41d..b5968e73d363a 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -34,7 +34,7 @@ import {ClusterSecretsManager} from './types/ClusterSecretsManager'; import ClusterLoginService from './components/cluster-login/service'; const State = { - DISCONNECTED: 'DISCONNECTED', + INIT: 'INIT', AGENT_DISCONNECTED: 'AGENT_DISCONNECTED', CLUSTER_DISCONNECTED: 'CLUSTER_DISCONNECTED', CONNECTED: 'CONNECTED' @@ -59,10 +59,9 @@ const SuccessStatus = { class ConnectionState { constructor(cluster) { - this.agents = []; this.cluster = cluster; this.clusters = []; - this.state = State.DISCONNECTED; + this.state = State.INIT; } updateCluster(cluster) { @@ -103,13 +102,11 @@ class ConnectionState { } disconnect() { - this.agents = []; - if (this.cluster) this.cluster.disconnect = true; this.clusters = []; - this.state = State.DISCONNECTED; + this.state = State.AGENT_DISCONNECTED; } } @@ -385,7 +382,11 @@ export default class AgentManager { } }); - this.$transitions.onExit({}, () => this.stopWatch()); + const stopWatchUnsubscribe = this.$transitions.onExit({}, () => { + this.stopWatch(); + + stopWatchUnsubscribe(); + }); return this.awaitCluster(); } diff --git a/modules/web-console/frontend/app/modules/user/User.service.js b/modules/web-console/frontend/app/modules/user/User.service.js index 3fdb9b9919fed..3515c1b96821b 100644 --- a/modules/web-console/frontend/app/modules/user/User.service.js +++ b/modules/web-console/frontend/app/modules/user/User.service.js @@ -15,6 +15,8 @@ * limitations under the License. */ +import {ReplaySubject} from 'rxjs/ReplaySubject'; + /** * @typedef User * @prop {string} _id @@ -40,7 +42,10 @@ export default function User($q, $injector, $root, $state, $http) { /** @type {ng.IPromise} */ let user; + const current$ = new ReplaySubject(1); + return { + current$, /** * @returns {ng.IPromise} */ @@ -51,6 +56,8 @@ export default function User($q, $injector, $root, $state, $http) { $root.$broadcast('user', $root.user); + current$.next(data); + return $root.user; }) .catch(({data}) => { diff --git a/modules/web-console/frontend/app/modules/user/user.module.js b/modules/web-console/frontend/app/modules/user/user.module.js index 9591a98cc3f17..3e3068fe948fc 100644 --- a/modules/web-console/frontend/app/modules/user/user.module.js +++ b/modules/web-console/frontend/app/modules/user/user.module.js @@ -35,7 +35,7 @@ function sessionRecoverer($injector, $q) { const stateName = $injector.get('$uiRouterGlobals').current.name; - if (!_.includes(['', 'signin'], stateName)) + if (!_.includes(['', 'signin', 'terms', '403', '404'], stateName)) $injector.get('$state').go('signin'); } @@ -76,25 +76,28 @@ function run($root, $transitions, AclService, User, Activities) { $transitions.onBefore({}, (trans) => { const $state = trans.router.stateService; - const {name, permission} = trans.to(); + const {permission} = trans.to(); if (_.isEmpty(permission)) return; return trans.injector().get('User').read() .then(() => { - if (AclService.can(permission)) { - Activities.post({action: $state.href(name, trans.params('to'))}); - - return; - } - - return $state.target(trans.to().failState || '403'); + if (!AclService.can(permission)) + throw new Error('Illegal access error'); }) .catch(() => { return $state.target(trans.to().failState || '403'); }); }); + + $transitions.onFinish({}, (trans) => { + const $state = trans.router.stateService; + const {name, permission} = trans.to(); + + if (AclService.can(permission)) + Activities.post({action: $state.href(name, trans.params('to'))}); + }); } run.$inject = ['$rootScope', '$transitions', 'AclService', 'User', 'IgniteActivitiesData']; diff --git a/modules/web-console/frontend/app/primitives/tooltip/index.scss b/modules/web-console/frontend/app/primitives/tooltip/index.scss index a6cd79ff327d3..71076e26c9eef 100644 --- a/modules/web-console/frontend/app/primitives/tooltip/index.scss +++ b/modules/web-console/frontend/app/primitives/tooltip/index.scss @@ -39,4 +39,4 @@ border-color: #f34718; color: white; } -} \ No newline at end of file +} diff --git a/modules/web-console/frontend/app/services/FormUtils.service.js b/modules/web-console/frontend/app/services/FormUtils.service.js index c8ce4b04f448d..aa4638194e1aa 100644 --- a/modules/web-console/frontend/app/services/FormUtils.service.js +++ b/modules/web-console/frontend/app/services/FormUtils.service.js @@ -339,7 +339,7 @@ export default function service($window, Focus, $rootScope) { return; const walk = (m) => { - if (!m.$error[e]) + if (!m || !m.$error[e]) return; if (m.$error[e] === true) diff --git a/modules/web-console/frontend/app/services/exceptionHandler.js b/modules/web-console/frontend/app/services/exceptionHandler.js index c6a09c3324495..6b5f4e472ea70 100644 --- a/modules/web-console/frontend/app/services/exceptionHandler.js +++ b/modules/web-console/frontend/app/services/exceptionHandler.js @@ -25,6 +25,10 @@ export function $exceptionHandler($log) { if (exception instanceof CancellationError) return; + // From ui-grid + if (exception === 'Possibly unhandled rejection: canceled') + return; + $log.error(exception, cause); }; } diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json index fce79ef5dbbd0..2260f75a4e5d3 100644 --- a/modules/web-console/frontend/package.json +++ b/modules/web-console/frontend/package.json @@ -10,7 +10,7 @@ "build": "webpack --config ./webpack/webpack.prod.js", "test": "karma start ./test/karma.conf.js", "test-watch": "npm test -- --no-single-run", - "eslint": "eslint --format node_modules/eslint-friendly-formatter app/ -- --eff-by-issue" + "eslint": "eslint --ignore-pattern node_modules/ --format node_modules/eslint-formatter-friendly . -- --eff-by-issue" }, "license": "Apache-2.0", "keywords": [ @@ -28,7 +28,6 @@ ], "dependencies": { "@babel/plugin-transform-parameters": "7.0.0", - "@types/angular-translate": "2.16.0", "@uirouter/angularjs": "1.0.20", "@uirouter/core": "5.0.19", "@uirouter/rx": "0.4.1", @@ -85,6 +84,7 @@ "@types/angular-animate": "1.5.10", "@types/angular-mocks": "1.5.12", "@types/angular-strap": "2.3.1", + "@types/angular-translate": "2.16.0", "@types/chai": "4.1.4", "@types/copy-webpack-plugin": "4.4.2", "@types/karma": "1.7.4", @@ -108,7 +108,6 @@ "eslint": "4.19.1", "eslint-formatter-friendly": "6.0.0", "eslint-loader": "2.1.0", - "eslint-plugin-babel": "5.2.0", "eslint-plugin-typescript": "0.12.0", "expose-loader": "0.7.5", "file-loader": "1.1.11", diff --git a/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss b/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss index ded958e4e6cc0..e68dedf70eeac 100644 --- a/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss +++ b/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss @@ -726,7 +726,7 @@ $list-group-link-heading-color: #333 !default; $panel-bg: #fff !default; $panel-body-padding: 15px !default; -$panel-heading-padding: 10px 15px !default; +$panel-heading-padding: 5px 10px !default; $panel-footer-padding: $panel-heading-padding !default; $panel-border-radius: $border-radius-base !default; diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss index a00a5857ba1bc..f9c9c4ad6a543 100644 --- a/modules/web-console/frontend/public/stylesheets/style.scss +++ b/modules/web-console/frontend/public/stylesheets/style.scss @@ -593,7 +593,6 @@ button.form-control { } .theme-line .panel-heading { - padding: 5px 10px; margin: 0; cursor: pointer; font-size: $font-size-large; From 637b6a1b92791038d9cd9a1c27a1fdbbe0a0e6c7 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Mon, 8 Oct 2018 11:44:22 +0700 Subject: [PATCH 118/403] IGNITE-9460 Web Console: Removed underline on top menu link focus. (cherry picked from commit 202f9f8dd077cd1129cb0c5b11ffebe0a9f4d00e) --- .../frontend/app/components/web-console-header/style.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/web-console/frontend/app/components/web-console-header/style.scss b/modules/web-console/frontend/app/components/web-console-header/style.scss index e92fdd5b66843..491078e6fd0ed 100644 --- a/modules/web-console/frontend/app/components/web-console-header/style.scss +++ b/modules/web-console/frontend/app/components/web-console-header/style.scss @@ -106,6 +106,7 @@ web-console-header { &:focus, :focus, &.active, .active { color: $color-active; + text-decoration: none; } &[disabled] { From fb57c5c59cb6179aa444fd1dc431409975483320 Mon Sep 17 00:00:00 2001 From: Alexey Platonov Date: Wed, 10 Oct 2018 10:53:31 +0300 Subject: [PATCH 119/403] IGNITE-9834: Cancel tcp-client-disco-msg-worker in normal order after validation error (cherry picked from commit 6259d3e) --- .../java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 23e2f4c251b8e..b0969dd3eb5b5 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -1884,6 +1884,7 @@ else if (log.isInfoEnabled()) { err = spi.duplicateIdError((TcpDiscoveryDuplicateIdMessage)msg); else if (discoMsg instanceof TcpDiscoveryAuthFailedMessage) err = spi.authenticationFailedError((TcpDiscoveryAuthFailedMessage)msg); + //TODO: https://issues.apache.org/jira/browse/IGNITE-9829 else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage) err = spi.checkFailedError((TcpDiscoveryCheckFailedMessage)msg); @@ -1898,6 +1899,7 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage) else joinError(err); + cancel(); break; } } From 31591f0384aec7bf4c6f47e9c3ce0b36387a5706 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 8 Oct 2018 14:00:41 +0700 Subject: [PATCH 120/403] IGNITE-9809 Fixed typo. (cherry picked from commit 81533dda0a09ece30041e3ebc5ae45089396b74f) --- .../frontend/app/components/web-console-header/style.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/web-console-header/style.scss b/modules/web-console/frontend/app/components/web-console-header/style.scss index 491078e6fd0ed..4edf6fd5e9318 100644 --- a/modules/web-console/frontend/app/components/web-console-header/style.scss +++ b/modules/web-console/frontend/app/components/web-console-header/style.scss @@ -30,7 +30,7 @@ web-console-header { position: relative; &:after { - // Shows header shadow over absoluteley positioned child content, + // Shows header shadow over absolutely positioned child content, // otherwise z ordering issues happen. display: block; width: 100%; From daf369c0ec03e5381fb486fd5de58069e6d38162 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Mon, 8 Oct 2018 17:37:42 +0700 Subject: [PATCH 121/403] IGNITE-9789 Web Console: Fixed issue with Roboto bold by preloading it. (cherry picked from commit 917a433239832e1cd384caf09d8f1fa967e9c944) --- modules/web-console/frontend/views/index.pug | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/web-console/frontend/views/index.pug b/modules/web-console/frontend/views/index.pug index b9ba00bea21a8..647f1a225ae23 100644 --- a/modules/web-console/frontend/views/index.pug +++ b/modules/web-console/frontend/views/index.pug @@ -28,6 +28,8 @@ html meta(name='fragment' content='!') + link(rel='preload' as='font' href='assets/fonts/roboto-bold-webfont.woff' type='font/woff' crossorigin='anonymous') + body.theme-line.body-overlap(ng-class='{ "demo-mode": IgniteDemoMode }') .splash.splash-max-foreground(hide-on-state-change) From 924510fa2b76ba79eb7bab794a76a490390e415b Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Tue, 9 Oct 2018 11:23:50 +0700 Subject: [PATCH 122/403] IGNITE-9596 Web Console: Fixed checkbox cross-browser visibility. (cherry picked from commit bc87ad9cb6f537bdcd0714d608ae0710691bd1d9) --- .../page-queries/components/queries-notebook/style.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss index fe0ede1b962e5..497870101dc67 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss @@ -124,7 +124,7 @@ queries-notebook { .form-field__sensitive { input[type='checkbox'] { - height: 0; + display: none; } input:checked + span { From 6957dc300c6ea1111237b8bc8eb7fd70de727dea Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 9 Oct 2018 14:47:26 +0700 Subject: [PATCH 123/403] IGNITE-9809 Web Console: Minor code style changes. (cherry picked from commit 149b6c1272b4a9d5250cc105e55b3fba9e97c60c) --- .../web-console/backend/app/agentSocket.js | 3 +- .../e2e/testcafe/testcafe-runner.js | 3 +- .../page-configure/services/ConfigureState.js | 3 +- .../app/components/page-landing/index.js | 3 +- .../queries-notebooks-list/controller.js | 42 +++++++++---------- .../app/modules/agent/AgentManager.service.js | 5 ++- .../test/check-doc-links/check-doc-links.js | 3 +- 7 files changed, 34 insertions(+), 28 deletions(-) diff --git a/modules/web-console/backend/app/agentSocket.js b/modules/web-console/backend/app/agentSocket.js index 0c7e6b2f78b9d..dfebda0b1b232 100644 --- a/modules/web-console/backend/app/agentSocket.js +++ b/modules/web-console/backend/app/agentSocket.js @@ -127,7 +127,8 @@ module.exports.factory = function() { const top = this.restResultParse(res); _.forEach(this.demo.browserSockets, (sock) => sock.emit('topology', top)); - } catch (err) { + } + catch (err) { _.forEach(this.demo.browserSockets, (sock) => sock.emit('topology:err', err)); } }); diff --git a/modules/web-console/e2e/testcafe/testcafe-runner.js b/modules/web-console/e2e/testcafe/testcafe-runner.js index 681ee2ed0e758..eab767c03d0d1 100644 --- a/modules/web-console/e2e/testcafe/testcafe-runner.js +++ b/modules/web-console/e2e/testcafe/testcafe-runner.js @@ -42,7 +42,8 @@ const startTestcafe = (config) => { .browsers(config.browsers) .reporter(config.reporter) .run({ skipJsErrors: true }); - } catch (err) { + } + catch (err) { console.log(err); process.exit(1); diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js index 27e227d8dce9b..ed911beb08381 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js +++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js @@ -30,7 +30,8 @@ export default class ConfigureState { const reducer = (state = {}, action) => { try { return this._combinedReducer(state, action); - } catch (e) { + } + catch (e) { console.error(e); return state; diff --git a/modules/web-console/frontend/app/components/page-landing/index.js b/modules/web-console/frontend/app/components/page-landing/index.js index ff6ee595950e7..8f678f397c1e4 100644 --- a/modules/web-console/frontend/app/components/page-landing/index.js +++ b/modules/web-console/frontend/app/components/page-landing/index.js @@ -54,7 +54,8 @@ export default angular const restored = trans.router.stateService.target(name, params); return restored.valid() ? restored : 'default-state'; - } catch (ignored) { + } + catch (ignored) { return 'default-state'; } }) diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js index 2e5ac39efd998..9b9b1c2eed3d7 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js @@ -98,11 +98,11 @@ export class NotebooksListCtrl { this.IgniteLoading.start('notebooksLoading'); this.notebooks = await this.IgniteNotebook.read(); this.gridOptions.data = this._preprocessNotebooksList(this.notebooks); - - } catch (err) { + } + catch (err) { this.IgniteMessages.showError(err); - - } finally { + } + finally { this.$scope.$applyAsync(); await this.IgniteLoading.finish('notebooksLoading'); @@ -135,18 +135,18 @@ export class NotebooksListCtrl { async createNotebook() { try { - const newNotebookName = await this.IgniteInput.input('New query notebook', 'Notebook name'); + const newNotebookName = await this.IgniteInput.input('New query notebook', 'Notebook name'); this.IgniteLoading.start('notebooksLoading'); await this.IgniteNotebook.create(newNotebookName); await this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); - - } catch (err) { + } + catch (err) { this.IgniteMessages.showError(err); - - } finally { + } + finally { await this.IgniteLoading.finish('notebooksLoading'); if (this.createNotebookModal) @@ -156,19 +156,19 @@ export class NotebooksListCtrl { async renameNotebok() { try { - const currentNotebook = this.gridApi.selection.legacyGetSelectedRows()[0]; - const newNotebookName = await this.IgniteInput.input('Rename notebook', 'Notebook name', currentNotebook.name); + const currentNotebook = this.gridApi.selection.legacyGetSelectedRows()[0]; + const newNotebookName = await this.IgniteInput.input('Rename notebook', 'Notebook name', currentNotebook.name); if (this.getNotebooksNames().find((name) => newNotebookName === name)) throw Error(`Notebook with name "${newNotebookName}" already exists!`); this.IgniteLoading.start('notebooksLoading'); await this.IgniteNotebook.save(Object.assign(currentNotebook, {name: newNotebookName})); - - } catch (err) { + } + catch (err) { this.IgniteMessages.showError(err); - - } finally { + } + finally { await this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); } @@ -184,11 +184,11 @@ export class NotebooksListCtrl { await this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); - - } catch (err) { + } + catch (err) { this.IgniteMessages.showError(err); - - } finally { + } + finally { await this.IgniteLoading.finish('notebooksLoading'); if (this.createNotebookModal) @@ -207,8 +207,8 @@ export class NotebooksListCtrl { await this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); - - } catch (err) { + } + catch (err) { this.IgniteMessages.showError(err); await this.IgniteLoading.finish('notebooksLoading'); diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index b5968e73d363a..4312785e0ffa1 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -18,7 +18,7 @@ import _ from 'lodash'; import {nonEmpty, nonNil} from 'app/utils/lodashMixins'; -import { BehaviorSubject } from 'rxjs/BehaviorSubject'; +import {BehaviorSubject} from 'rxjs/BehaviorSubject'; import 'rxjs/add/operator/first'; import 'rxjs/add/operator/partition'; import 'rxjs/add/operator/takeUntil'; @@ -239,7 +239,8 @@ export default class AgentManager { saveToStorage(cluster = this.connectionSbj.getValue().cluster) { try { localStorage.cluster = JSON.stringify(cluster); - } catch (ignore) { + } + catch (ignore) { // No-op. } } diff --git a/modules/web-console/frontend/test/check-doc-links/check-doc-links.js b/modules/web-console/frontend/test/check-doc-links/check-doc-links.js index e8f3d169904ea..4ab64ed2d4431 100644 --- a/modules/web-console/frontend/test/check-doc-links/check-doc-links.js +++ b/modules/web-console/frontend/test/check-doc-links/check-doc-links.js @@ -66,7 +66,8 @@ const findLinks = (acc, ast) => { try { acc.push([JSON.parse(compiledAttr).href, ast.filePath]); - } catch (e) { + } + catch (e) { console.log(ast.filePath, e); } } From 20edc258f823c3ddf3b2dca67646bc7a80586a7e Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Wed, 10 Oct 2018 11:29:23 +0700 Subject: [PATCH 124/403] IGNITE-9809 Add extra methods to E2E FormField component. (cherry picked from commit 9afea90b9e36800bcf577b8aa8c58148532acd0e) --- .../e2e/testcafe/components/FormField.js | 14 ++++++++++++++ .../frontend/app/modules/agent/types/Cluster.js | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/web-console/e2e/testcafe/components/FormField.js b/modules/web-console/e2e/testcafe/components/FormField.js index 4ddfc6199124f..b0e89675103da 100644 --- a/modules/web-console/e2e/testcafe/components/FormField.js +++ b/modules/web-console/e2e/testcafe/components/FormField.js @@ -63,6 +63,12 @@ export class FormField { // return this._selector.find(`.form-field__error`) return this._selector.find(`[ng-message="${errorType}"]`); } + get selectedOption() { + return this.control.textContent; + } + get postfix() { + return this._selector.find('[data-postfix]').getAttribute('data-postfix'); + } } /** @@ -72,4 +78,12 @@ export class CustomFormField extends FormField { static ROOT_SELECTOR = '.form-field'; static LABEL_SELECTOR = '.form-field__label'; static ERRORS_SELECTOR = '.form-field__errors'; + constructor(...args) { + super(...args); + this.errors = this.errors.addCustomMethods({ + hasError(errors, errorMessage) { + return !!errors.querySelectorAll(`.form-field__error [data-title*="${errorMessage}"]`).length; + } + }); + } } diff --git a/modules/web-console/frontend/app/modules/agent/types/Cluster.js b/modules/web-console/frontend/app/modules/agent/types/Cluster.js index dd054050e714f..efb303d9b6caa 100644 --- a/modules/web-console/frontend/app/modules/agent/types/Cluster.js +++ b/modules/web-console/frontend/app/modules/agent/types/Cluster.js @@ -15,7 +15,7 @@ * limitations under the License. */ -class Cluster { +export class Cluster { /** @type {String} */ id; From 11f42133310d85fef1ae9ec2de4365d215b63165 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Thu, 11 Oct 2018 11:39:29 +0700 Subject: [PATCH 125/403] IGNITE-9809 Web Console: Added possibility to disable multiselect filter in ui-grid column definitions. (cherry picked from commit bc62ed44d50f9ebf8e1a873f96a7aeeecb86a6ca) --- .../frontend/app/components/ui-grid-filters/template.pug | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/template.pug b/modules/web-console/frontend/app/components/ui-grid-filters/template.pug index 4cdcc8705f6a2..a865395bdd34e 100644 --- a/modules/web-console/frontend/app/components/ui-grid-filters/template.pug +++ b/modules/web-console/frontend/app/components/ui-grid-filters/template.pug @@ -27,6 +27,7 @@ type='button' title='{{ colFilter.$$multiselectFilterTooltip() }}' ng-model='colFilter.term' + ng-disabled='col.colDef.multiselectFilterDisabled' bs-select bs-options='option.value as option.label for option in colFilter.selectOptions' data-multiple='true' @@ -41,7 +42,7 @@ }` ng-click='colFilter.selectDialog(grid, colFilter)' type='button' - title='{{ colFilter.$$multiselectFilterTooltip() }}' + title='{{ colFilter.$$multiselectFilterTooltip() }}' ) {{ col.displayName }} .ui-grid-cell-contents(role='button') From 197d5ff9021a71ca1f8783d17c4a8443abae88a6 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Thu, 11 Oct 2018 13:55:11 +0700 Subject: [PATCH 126/403] IGNITE-9846 Web Console: allow to $inject before function definition. (cherry picked from commit 824939ba1f238621bd89a5ff445c6932c79e5db9) --- modules/web-console/frontend/.eslintrc | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/web-console/frontend/.eslintrc b/modules/web-console/frontend/.eslintrc index ac1a37f55df81..8f6bf20970b31 100644 --- a/modules/web-console/frontend/.eslintrc +++ b/modules/web-console/frontend/.eslintrc @@ -163,7 +163,6 @@ rules: no-unused-expressions: [2, { allowShortCircuit: true }] no-unused-vars: [0, {"vars": "all", "args": "after-used"}] typescript/no-unused-vars: [0] - no-use-before-define: 2 no-useless-call: 2 no-void: 0 no-var: 2 From 76f0e6c3fcaa35b4d3d96edbf82f65d186d5174e Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Mon, 15 Oct 2018 10:22:21 +0700 Subject: [PATCH 127/403] IGNITE-9809 Web console: more minor fixes - Fixes #4972. Update TestCafe to 0.22.0. Update confirmation E2E selectors so these work when multiple modals are opened. Update reduxDevtoolsIntegration.js so it doesn't crash when there's a Promise in action payload. Fix form-field postfix position in browsers that don't support place-self. (cherry picked from commit 11716dedca0c884bfe9b6afdc9492ed0a3c6a58a) --- modules/web-console/e2e/testcafe/components/confirmation.js | 4 ++-- modules/web-console/e2e/testcafe/package.json | 2 +- .../components/page-configure/reduxDevtoolsIntegration.js | 5 +++++ .../frontend/app/primitives/form-field/index.scss | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/web-console/e2e/testcafe/components/confirmation.js b/modules/web-console/e2e/testcafe/components/confirmation.js index b4fa2b5aa2719..cdfdd548415ff 100644 --- a/modules/web-console/e2e/testcafe/components/confirmation.js +++ b/modules/web-console/e2e/testcafe/components/confirmation.js @@ -17,10 +17,10 @@ import {Selector, t} from 'testcafe'; -const body = Selector('.modal-body'); +const body = Selector('.modal').withText('Confirmation').find('.modal-body'); const confirmButton = Selector('#confirm-btn-ok'); const cancelButton = Selector('#confirm-btn-cancel'); -const closeButton = Selector('.modal .close'); +const closeButton = Selector('.modal').withText('Confirmation').find('.modal .close'); export const confirmation = { body, diff --git a/modules/web-console/e2e/testcafe/package.json b/modules/web-console/e2e/testcafe/package.json index 23d1b2707e1a5..6cba09e14020d 100644 --- a/modules/web-console/e2e/testcafe/package.json +++ b/modules/web-console/e2e/testcafe/package.json @@ -33,7 +33,7 @@ "objectid": "3.2.1", "path": "0.12.7", "sinon": "2.3.8", - "testcafe": "0.20.5", + "testcafe": "0.22.0", "testcafe-angular-selectors": "0.3.0", "testcafe-reporter-teamcity": "1.0.9", "type-detect": "4.0.3", diff --git a/modules/web-console/frontend/app/components/page-configure/reduxDevtoolsIntegration.js b/modules/web-console/frontend/app/components/page-configure/reduxDevtoolsIntegration.js index 43fa3237ffcf6..f602539d1140e 100644 --- a/modules/web-console/frontend/app/components/page-configure/reduxDevtoolsIntegration.js +++ b/modules/web-console/frontend/app/components/page-configure/reduxDevtoolsIntegration.js @@ -36,6 +36,11 @@ const replacer = (key, value) => { __serializedType__: 'Symbol' }; } + if (value instanceof Promise) { + return { + data: {} + }; + } return value; }; diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss index 0330e5886a42a..3e757f9eecca9 100644 --- a/modules/web-console/frontend/app/primitives/form-field/index.scss +++ b/modules/web-console/frontend/app/primitives/form-field/index.scss @@ -259,7 +259,7 @@ &--postfix::after { content: attr(data-postfix); display: inline-flex; - place-self: center; + align-self: center; margin-left: 10px; } From 5e744bcc912cc1a63fcd1a8c8eacc7f6c555cc59 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Mon, 15 Oct 2018 15:12:37 +0700 Subject: [PATCH 128/403] IGNITE-9059 Web Console: Removed not needed "await" on loader. (cherry picked from commit 55e034ab6a3f2e34d8eb3902e41f47dcb8ec775b) --- .../queries-notebooks-list/controller.js | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js index 9b9b1c2eed3d7..379f97453b934 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js @@ -105,7 +105,7 @@ export class NotebooksListCtrl { finally { this.$scope.$applyAsync(); - await this.IgniteLoading.finish('notebooksLoading'); + this.IgniteLoading.finish('notebooksLoading'); } } @@ -138,8 +138,10 @@ export class NotebooksListCtrl { const newNotebookName = await this.IgniteInput.input('New query notebook', 'Notebook name'); this.IgniteLoading.start('notebooksLoading'); + await this.IgniteNotebook.create(newNotebookName); - await this.IgniteLoading.finish('notebooksLoading'); + + this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); } @@ -147,7 +149,7 @@ export class NotebooksListCtrl { this.IgniteMessages.showError(err); } finally { - await this.IgniteLoading.finish('notebooksLoading'); + this.IgniteLoading.finish('notebooksLoading'); if (this.createNotebookModal) this.createNotebookModal.$promise.then(this.createNotebookModal.hide); @@ -163,13 +165,15 @@ export class NotebooksListCtrl { throw Error(`Notebook with name "${newNotebookName}" already exists!`); this.IgniteLoading.start('notebooksLoading'); + await this.IgniteNotebook.save(Object.assign(currentNotebook, {name: newNotebookName})); } catch (err) { this.IgniteMessages.showError(err); } finally { - await this.IgniteLoading.finish('notebooksLoading'); + this.IgniteLoading.finish('notebooksLoading'); + this._loadAllNotebooks(); } } @@ -180,8 +184,10 @@ export class NotebooksListCtrl { const newNotebookName = await this.IgniteInput.clone(clonedNotebook.name, this.getNotebooksNames()); this.IgniteLoading.start('notebooksLoading'); + await this.IgniteNotebook.clone(newNotebookName, clonedNotebook); - await this.IgniteLoading.finish('notebooksLoading'); + + this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); } @@ -189,7 +195,7 @@ export class NotebooksListCtrl { this.IgniteMessages.showError(err); } finally { - await this.IgniteLoading.finish('notebooksLoading'); + this.IgniteLoading.finish('notebooksLoading'); if (this.createNotebookModal) this.createNotebookModal.$promise.then(this.createNotebookModal.hide); @@ -203,15 +209,17 @@ export class NotebooksListCtrl { async deleteNotebooks() { try { this.IgniteLoading.start('notebooksLoading'); + await this.IgniteNotebook.removeBatch(this.gridApi.selection.legacyGetSelectedRows()); - await this.IgniteLoading.finish('notebooksLoading'); + + this.IgniteLoading.finish('notebooksLoading'); this._loadAllNotebooks(); } catch (err) { this.IgniteMessages.showError(err); - await this.IgniteLoading.finish('notebooksLoading'); + this.IgniteLoading.finish('notebooksLoading'); } } From 88bf0635f3dea3c0635c1f278d2abb4e51e38094 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Tue, 16 Oct 2018 11:27:00 +0700 Subject: [PATCH 129/403] IGNITE-9748 Web Console: Added suport for MVCC on "Configuration" screens. (cherry picked from commit a800c36f1603f1c206d3847a04e0f9eb22ae547f) --- .../visor/VisorDataTransferObject.java | 3 + .../visor/node/VisorGridConfiguration.java | 22 ++++- .../visor/node/VisorMvccConfiguration.java | 94 +++++++++++++++++++ modules/web-console/backend/app/schemas.js | 6 +- .../cache-edit-form/templates/general.pug | 18 ++-- .../cluster-edit-form/template.tpl.pug | 3 + .../cluster-edit-form/templates/mvcc.pug | 46 +++++++++ .../page-configure-basic/controller.js | 3 +- .../app/components/page-configure/style.scss | 1 + .../generator/AbstractTransformer.js | 5 + .../generator/ConfigurationGenerator.js | 12 ++- .../frontend/app/services/Caches.js | 3 +- 12 files changed, 200 insertions(+), 16 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMvccConfiguration.java create mode 100644 modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/mvcc.pug diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java index 6ba3aa7d5aed5..6fd51cdd3d55c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java @@ -50,6 +50,9 @@ public abstract class VisorDataTransferObject implements Externalizable { /** Version 3. */ protected static final byte V3 = 3; + /** Version 4. */ + protected static final byte V4 = 4; + /** * @param col Source collection. * @param Collection type. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java index 85849a56f907c..a9144abc5f45a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java @@ -126,6 +126,9 @@ public class VisorGridConfiguration extends VisorDataTransferObject { /** Client connector configuration */ private VisorClientConnectorConfiguration clnConnCfg; + /** MVCC configuration. */ + private VisorMvccConfiguration mvccCfg; + /** * Default constructor. */ @@ -192,6 +195,8 @@ public VisorGridConfiguration(IgniteEx ignite) { if (dsCfg != null) dataStorage = new VisorDataStorageConfiguration(dsCfg); + + mvccCfg = new VisorMvccConfiguration(c); } /** @@ -383,9 +388,16 @@ public VisorDataStorageConfiguration getDataStorageConfiguration() { return dataStorage; } + /** + * @return MVCC configuration. + */ + public VisorMvccConfiguration getMvccConfiguration() { + return mvccCfg; + } + /** {@inheritDoc} */ @Override public byte getProtocolVersion() { - return V3; + return V4; } /** {@inheritDoc} */ @@ -417,6 +429,7 @@ public VisorDataStorageConfiguration getDataStorageConfiguration() { U.writeCollection(out, srvcCfgs); out.writeObject(dataStorage); out.writeObject(clnConnCfg); + out.writeObject(mvccCfg); } /** {@inheritDoc} */ @@ -447,11 +460,14 @@ public VisorDataStorageConfiguration getDataStorageConfiguration() { sqlConnCfg = (VisorSqlConnectorConfiguration) in.readObject(); srvcCfgs = U.readList(in); - if (protoVer >= V2) + if (protoVer > V1) dataStorage = (VisorDataStorageConfiguration)in.readObject(); - if (protoVer >= V3) + if (protoVer > V2) clnConnCfg = (VisorClientConnectorConfiguration)in.readObject(); + + if (protoVer > V3) + mvccCfg = (VisorMvccConfiguration)in.readObject(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMvccConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMvccConfiguration.java new file mode 100644 index 0000000000000..1bcaa21507818 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMvccConfiguration.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.node; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * Data transfer object for data store configuration. + */ +public class VisorMvccConfiguration extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** Number of MVCC vacuum cleanup threads. */ + private int mvccVacuumThreadCnt; + + /** Time interval between vacuum runs */ + private long mvccVacuumFreq; + + /** + * Default constructor. + */ + public VisorMvccConfiguration() { + // No-op. + } + + /** + * Constructor. + * + * @param cfg Ignite configuration. + */ + public VisorMvccConfiguration(IgniteConfiguration cfg) { + assert cfg != null; + + mvccVacuumThreadCnt = cfg.getMvccVacuumThreadCount(); + mvccVacuumFreq = cfg.getMvccVacuumFrequency(); + } + + /** + * @return Number of MVCC vacuum threads. + */ + public int getMvccVacuumThreadCount() { + return mvccVacuumThreadCnt; + } + + /** + * @return Time interval between MVCC vacuum runs in milliseconds. + */ + public long getMvccVacuumFrequency() { + return mvccVacuumFreq; + } + + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V1; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + out.writeInt(mvccVacuumThreadCnt); + out.writeLong(mvccVacuumFreq); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + mvccVacuumThreadCnt = in.readInt(); + mvccVacuumFreq = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorMvccConfiguration.class, this); + } +} diff --git a/modules/web-console/backend/app/schemas.js b/modules/web-console/backend/app/schemas.js index 5898ed88cb247..cf174f7ad3086 100644 --- a/modules/web-console/backend/app/schemas.js +++ b/modules/web-console/backend/app/schemas.js @@ -136,7 +136,7 @@ module.exports.factory = function(mongoose) { clusters: [{type: ObjectId, ref: 'Cluster'}], domains: [{type: ObjectId, ref: 'DomainModel'}], cacheMode: {type: String, enum: ['PARTITIONED', 'REPLICATED', 'LOCAL']}, - atomicityMode: {type: String, enum: ['ATOMIC', 'TRANSACTIONAL']}, + atomicityMode: {type: String, enum: ['ATOMIC', 'TRANSACTIONAL', 'TRANSACTIONAL_SNAPSHOT']}, partitionLossPolicy: { type: String, enum: ['READ_ONLY_SAFE', 'READ_ONLY_ALL', 'READ_WRITE_SAFE', 'READ_WRITE_ALL', 'IGNORE'] @@ -1109,7 +1109,9 @@ module.exports.factory = function(mongoose) { rateTimeInterval: Number, tlbSize: Number, subIntervals: Number - } + }, + mvccVacuumThreadCount: Number, + mvccVacuumFrequency: Number }); Cluster.index({name: 1, space: 1}, {unique: true}); diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug index 955239693354a..ba5d5ad3c2f3d 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug @@ -74,16 +74,18 @@ panel-collapsible(opened=`::true` ng-form=form) name: '"atomicityMode"', placeholder: 'ATOMIC', options: '[\ - {value: "ATOMIC", label: "ATOMIC"},\ - {value: "TRANSACTIONAL", label: "TRANSACTIONAL"}\ - ]', + {value: "ATOMIC", label: "ATOMIC"},\ + {value: "TRANSACTIONAL", label: "TRANSACTIONAL"},\ + {value: "TRANSACTIONAL_SNAPSHOT", label: "TRANSACTIONAL_SNAPSHOT"}\ + ]', tip: 'Atomicity:\ -
    \ -
  • ATOMIC - in this mode distributed transactions and distributed locking are not supported
  • \ -
  • TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior
  • \ -
' +
    \ +
  • ATOMIC - in this mode distributed transactions and distributed locking are not supported
  • \ +
  • TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior
  • \ +
  • TRANSACTIONAL_SNAPSHOT - in this mode specified fully ACID-compliant transactional cache behavior for both key-value API and SQL transactions
  • \ +
' }) - .pc-form-grid-col-30(ng-is=`${model}.cacheMode === 'PARTITIONED'`) + .pc-form-grid-col-30(ng-if=`${model}.cacheMode === 'PARTITIONED'`) +form-field__number({ label: 'Backups:', model: `${model}.backups`, diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug index d5cb90930f8c2..e30cfdb7b5589 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug @@ -50,6 +50,9 @@ form(id='cluster' name='ui.inputForm' novalidate) include ./templates/misc include ./templates/metrics + //- Since ignite 2.7 + include ./templates/mvcc + //- Deprecated in ignite 2.1 include ./templates/odbc diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/mvcc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/mvcc.pug new file mode 100644 index 0000000000000..2799520e2e614 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/mvcc.pug @@ -0,0 +1,46 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +include /app/helpers/jade/mixins + +-var form = 'mvcc' +-var model = '$ctrl.clonedCluster' + +panel-collapsible(ng-show='$ctrl.available("2.7.0")' ng-form=form on-open=`ui.loadPanel('${form}')`) + panel-title Multiversion concurrency control (MVCC) + panel-description Multiversion concurrency control (MVCC) configuration. + panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`) + .pca-form-column-6.pc-form-grid-row + .pc-form-grid-col-30(ng-if='$ctrl.available("2.1.0")') + +form-field__number({ + label: 'Vacuum thread pool size:', + model: `${model}.mvccVacuumThreadCount`, + name: '"MvccVacuumThreadCount"', + placeholder: '2', + min: '0', + tip: 'Number of MVCC vacuum cleanup threads' + }) + .pc-form-grid-col-30(ng-if='$ctrl.available("2.1.0")') + +form-field__number({ + label: 'Vacuum intervals:', + model: `${model}.mvccVacuumFrequency`, + name: '"MvccVacuumFrequency"', + placeholder: '5000', + min: '0', + tip: 'Time interval between vacuum runs in ms' + }) + .pca-form-column-6 + +preview-xml-java(model, 'clusterMvcc') diff --git a/modules/web-console/frontend/app/components/page-configure-basic/controller.js b/modules/web-console/frontend/app/components/page-configure-basic/controller.js index 88c61636f07ad..f5515fb99d2f3 100644 --- a/modules/web-console/frontend/app/components/page-configure-basic/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-basic/controller.js @@ -155,11 +155,12 @@ export default class PageConfigureBasicController { this.cachesColDefs = [ {name: 'Name:', cellClass: 'pc-form-grid-col-10'}, {name: 'Mode:', cellClass: 'pc-form-grid-col-10'}, - {name: 'Atomicity:', cellClass: 'pc-form-grid-col-10', tip: ` + {name: 'Atomicity:', cellClass: 'pc-form-grid-col-20', tip: ` Atomicity:
  • ATOMIC - in this mode distributed transactions and distributed locking are not supported
  • TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior
  • +
  • TRANSACTIONAL_SNAPSHOT - in this mode specified fully ACID-compliant transactional cache behavior for both key-value API and SQL transactions
`}, {name: 'Backups:', cellClass: 'pc-form-grid-col-10', tip: ` diff --git a/modules/web-console/frontend/app/components/page-configure/style.scss b/modules/web-console/frontend/app/components/page-configure/style.scss index 365e058cbbe31..9e9661cd2c1a8 100644 --- a/modules/web-console/frontend/app/components/page-configure/style.scss +++ b/modules/web-console/frontend/app/components/page-configure/style.scss @@ -222,6 +222,7 @@ list-editable .pc-form-group__text-title { &>.pc-form-grid-col-10 { flex-basis: calc(25%); } + &>.pc-form-grid-col-20 { flex-basis: calc(50%); } diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js index e0aece76a810c..ab69bb8fb638a 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js @@ -153,6 +153,11 @@ export default class AbstractTransformer { return this.toSection(this.generator.clusterMisc(cluster, available)); } + // Generate marshaller group. + static clusterMvcc(cluster, available) { + return this.toSection(this.generator.clusterMvcc(cluster, available)); + } + // Generate marshaller group. static clusterMarshaller(cluster, available) { return this.toSection(this.generator.clusterMarshaller(cluster, available)); diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js index 2a1a506d8b939..d80967e2cfcdc 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js @@ -107,6 +107,7 @@ export default class IgniteConfigurationGenerator { this.clusterMisc(cluster, available, cfg); this.clusterMetrics(cluster, available, cfg); + this.clusterMvcc(cluster, available, cfg); this.clusterODBC(cluster.odbc, available, cfg); // Since ignite 2.1 deprecated in ignite 2.3 @@ -1516,7 +1517,6 @@ export default class IgniteConfigurationGenerator { static clusterMisc(cluster, available, cfg = this.igniteConfigurationBean(cluster)) { cfg.stringProperty('workDirectory'); - // Since Ignite 2.0 if (available('2.0.0')) { cfg.stringProperty('consistentId') .emptyBeanProperty('warmupClosure') @@ -1530,6 +1530,16 @@ export default class IgniteConfigurationGenerator { return cfg; } + // Generate MVCC configuration. + static clusterMvcc(cluster, available, cfg = this.igniteConfigurationBean(cluster)) { + if (available('2.7.0')) { + cfg.intProperty('mvccVacuumThreadCount') + .intProperty('mvccVacuumFrequency'); + } + + return cfg; + } + // Generate IGFSs configs. static clusterIgfss(igfss, available, cfg = this.igniteConfigurationBean()) { const igfsCfgs = _.map(igfss, (igfs) => { diff --git a/modules/web-console/frontend/app/services/Caches.js b/modules/web-console/frontend/app/services/Caches.js index 7df4bb6ec13d8..626395d7bf9d6 100644 --- a/modules/web-console/frontend/app/services/Caches.js +++ b/modules/web-console/frontend/app/services/Caches.js @@ -31,7 +31,8 @@ export default class Caches { /** @type {ig.menu} */ atomicityModes = [ {value: 'ATOMIC', label: 'ATOMIC'}, - {value: 'TRANSACTIONAL', label: 'TRANSACTIONAL'} + {value: 'TRANSACTIONAL', label: 'TRANSACTIONAL'}, + {value: 'TRANSACTIONAL_SNAPSHOT', label: 'TRANSACTIONAL_SNAPSHOT'} ]; /** From 2fb063550a8383559ed2f5a38f06551c79446850 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Tue, 16 Oct 2018 17:30:12 +0700 Subject: [PATCH 130/403] IGNITE-9680 Web console: Added component for status output. (cherry picked from commit 5375042e987adc8d5cf489b24d6a4e09cdcf1714) --- modules/web-console/frontend/app/app.js | 4 +- .../components/cell-status/index.ts} | 22 ++++---- .../components/list/column-defs.js | 2 +- .../index.js => status-output/component.ts} | 15 ++++-- .../componentFactory.ts} | 16 ++++-- .../components/status-output/controller.ts | 53 +++++++++++++++++++ .../app/components/status-output/index.ts | 40 ++++++++++++++ .../cell-status => status-output}/style.scss | 34 +++++------- .../template.tpl.pug} | 4 +- .../app/components/ui-grid/controller.js | 4 ++ 10 files changed, 150 insertions(+), 44 deletions(-) rename modules/web-console/frontend/app/components/{ignite-status/style.scss => connected-clusters-dialog/components/cell-status/index.ts} (72%) rename modules/web-console/frontend/app/components/{connected-clusters-dialog/components/cell-status/index.js => status-output/component.ts} (78%) rename modules/web-console/frontend/app/components/{ignite-status/index.js => status-output/componentFactory.ts} (72%) create mode 100644 modules/web-console/frontend/app/components/status-output/controller.ts create mode 100644 modules/web-console/frontend/app/components/status-output/index.ts rename modules/web-console/frontend/app/components/{connected-clusters-dialog/components/cell-status => status-output}/style.scss (63%) rename modules/web-console/frontend/app/components/{connected-clusters-dialog/components/cell-status/template.pug => status-output/template.tpl.pug} (84%) diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index 98792c4c50bb8..c890fa42db588 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -148,7 +148,7 @@ import progressLine from './components/progress-line'; import formField from './components/form-field'; import igniteChart from './components/ignite-chart'; import igniteChartSelector from './components/ignite-chart-series-selector'; -import igniteStatus from './components/ignite-status'; +import statusOutput from './components/status-output'; import pageProfile from './components/page-profile'; import pagePasswordChanged from './components/page-password-changed'; @@ -258,7 +258,7 @@ export default angular.module('ignite-console', [ passwordVisibility.name, igniteChart.name, igniteChartSelector.name, - igniteStatus.name, + statusOutput.name, progressLine.name, formField.name ]) diff --git a/modules/web-console/frontend/app/components/ignite-status/style.scss b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.ts similarity index 72% rename from modules/web-console/frontend/app/components/ignite-status/style.scss rename to modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.ts index d2877fc5a71db..c3f201f15b37a 100644 --- a/modules/web-console/frontend/app/components/ignite-status/style.scss +++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.ts @@ -15,13 +15,17 @@ * limitations under the License. */ -@import "../../../public/stylesheets/variables"; +import {componentFactory, StatusLevel} from 'app/components/status-output'; -// Statuses coloring -.ignite-status__active { - color: $ignite-status-active !important; -} - -.ignite-status__inactive { - color: $ignite-status-inactive; -} +export default componentFactory([ + { + level: StatusLevel.GREEN, + value: true, + label: 'Active' + }, + { + level: StatusLevel.RED, + value: false, + label: 'Not Active' + } +]); diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js index ca5dc6b23f04e..6b25878d3e874 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js +++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js @@ -46,7 +46,7 @@ export default [ cellTemplate: `
({ + ...component, + bindings: { + value: '<' + }, + controller: class extends Status { + options = options + } +}); diff --git a/modules/web-console/frontend/app/components/status-output/controller.ts b/modules/web-console/frontend/app/components/status-output/controller.ts new file mode 100644 index 0000000000000..e9baab1dbcf38 --- /dev/null +++ b/modules/web-console/frontend/app/components/status-output/controller.ts @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {StatusOptions, StatusOption} from './index'; + +interface Changes extends ng.IOnChangesObject { + value: ng.IChangesObject, + options: ng.IChangesObject +} + +const UNIVERSAL_CLASSNAME = 'status-output'; + +export class Status implements ng.IComponentController, ng.IOnChanges, ng.IPostLink, ng.IOnDestroy { + static $inject = ['$element']; + + value: string; + options: StatusOptions; + status: StatusOption | undefined; + statusClassName: string | undefined; + + constructor(private el: JQLite) {} + + $postLink() { + this.el[0].classList.add(UNIVERSAL_CLASSNAME); + } + + $onDestroy() { + delete this.el; + } + + $onChanges(changes: Changes) { + if ('value' in changes) { + this.status = this.options.find((option) => option.value === this.value); + + if (this.status) + this.statusClassName = `${UNIVERSAL_CLASSNAME}__${this.status.level.toLowerCase()}`; + } + } +} diff --git a/modules/web-console/frontend/app/components/status-output/index.ts b/modules/web-console/frontend/app/components/status-output/index.ts new file mode 100644 index 0000000000000..c24d438762411 --- /dev/null +++ b/modules/web-console/frontend/app/components/status-output/index.ts @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {component} from './component'; +export {componentFactory} from './componentFactory'; + +export interface StatusOption { + level: StatusLevel, + // Whether to show progress indicator or not + ongoing?: boolean, + value: string | boolean, + // What user will see + label: string +} + +export type StatusOptions = Array; + +export enum StatusLevel { + NEUTRAL = 'NEUTRAL', + GREEN = 'GREEN', + RED = 'RED' +} + +export default angular + .module('ignite-console.components.status-output', []) + .component('statusOutput', component); diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss b/modules/web-console/frontend/app/components/status-output/style.scss similarity index 63% rename from modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss rename to modules/web-console/frontend/app/components/status-output/style.scss index 2f0c98149d1bc..d49213f2eb9d3 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss +++ b/modules/web-console/frontend/app/components/status-output/style.scss @@ -15,32 +15,24 @@ * limitations under the License. */ -connected-clusters-cell-status { - $color-active: #417505; - $color-not-active: #ee2b27; +.status-output { + @import 'public/stylesheets/variables'; - display: block; + display: inline-flex; - div { - display: flex; - - &:before { - content: '●'; + .spinner-circle { + margin-left: 5px; + } - position: relative; - height: 16px; - margin-right: 10px; - - font-size: 24px; - line-height: 19px; - } + .status-output__neutral { + color: inherit; } - .#{&}-active { - color: $color-active; + .status-output__green { + color: $ignite-status-active; } - .#{&}-not-active { - color: $color-not-active; + .status-output__red { + color: $ignite-status-inactive; } -} \ No newline at end of file +} diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug b/modules/web-console/frontend/app/components/status-output/template.tpl.pug similarity index 84% rename from modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug rename to modules/web-console/frontend/app/components/status-output/template.tpl.pug index e4f21b6c4b39a..a79035e6aa56e 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug +++ b/modules/web-console/frontend/app/components/status-output/template.tpl.pug @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. -div(ng-class='["connected-clusters-cell-status-"+($ctrl.status ? "active" : "not-active")]') - i {{ $ctrl.status ? 'Active' : 'Not Active' }} \ No newline at end of file +span(translate='{{$ctrl.status.label}}' ng-class='$ctrl.statusClassName') +.spinner-circle(ng-if='$ctrl.status.ongoing') diff --git a/modules/web-console/frontend/app/components/ui-grid/controller.js b/modules/web-console/frontend/app/components/ui-grid/controller.js index a6406044f10df..f53416b57e7f0 100644 --- a/modules/web-console/frontend/app/components/ui-grid/controller.js +++ b/modules/web-console/frontend/app/components/ui-grid/controller.js @@ -19,6 +19,9 @@ import debounce from 'lodash/debounce'; import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug'; export default class IgniteUiGrid { + /** @type {import('ui-grid').IGridOptions} */ + grid; + /** @type */ gridApi; @@ -75,6 +78,7 @@ export default class IgniteUiGrid { } this.grid = { + appScopeProvider: this.$scope.$parent, data: this.items, columnDefs: this.columnDefs, categories: this.categories, From f6f366e8c0ab858781d5159023e3a7482db94fc5 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 17 Oct 2018 15:43:52 +0700 Subject: [PATCH 131/403] IGNITE-9862 Minor fixes. (cherry picked from commit adcf3a39ab00a341e7d6fb99a04ed5bc9afc9e98) --- .../frontend/app/modules/agent/AgentManager.service.js | 3 +-- .../web-console/frontend/app/primitives/timepicker/index.pug | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index 4312785e0ffa1..7226cff988ecc 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -44,8 +44,7 @@ const IGNITE_2_0 = '2.0.0'; const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1']; const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], ['2.5.1-p13', '2.6.0'], '2.7.0']; -// Error codes from o.a.i.internal.processors.restGridRestResponse.java - +/** Error codes from o.a.i.internal.processors.restGridRestResponse.java */ const SuccessStatus = { /** Command succeeded. */ STATUS_SUCCESS: 0, diff --git a/modules/web-console/frontend/app/primitives/timepicker/index.pug b/modules/web-console/frontend/app/primitives/timepicker/index.pug index 5e3936c737c25..f97a5a2ce956a 100644 --- a/modules/web-console/frontend/app/primitives/timepicker/index.pug +++ b/modules/web-console/frontend/app/primitives/timepicker/index.pug @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. -mixin form-field__timepicker({ label, model, name, mindate, maxdate, disabled, required, placeholder, tip }) +mixin form-field__timepicker({ label, model, name, mindate, maxdate, disabled, required, placeholder, tip, format = 'HH:mm'}) mixin __form-field__timepicker() input( id=`{{ ${name} }}Input` @@ -28,9 +28,10 @@ mixin form-field__timepicker({ label, model, name, mindate, maxdate, disabled, r ng-disabled=disabled && `${disabled}` bs-timepicker - data-time-format='HH:mm' + data-time-format=format data-length='1' data-minute-step='1' + data-second-step='1' data-arrow-behavior='picker' data-placement='bottom' From fdf447f551c46f2b4fc9a389ca2a0e20f9efac41 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 25 Oct 2018 10:37:36 +0700 Subject: [PATCH 132/403] IGNITE-9938 Updated ScalaTest version and introduced "${scala.test.version}" variable. (cherry picked from commit e7a93fcc1383003596370aa65efdce053f0f2b48) --- examples/pom.xml | 2 +- modules/scalar/pom.xml | 2 +- modules/spark/pom.xml | 2 +- modules/visor-console/pom.xml | 2 +- parent/pom.xml | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/pom.xml b/examples/pom.xml index e745beb7204d4..c6b0a5f59db0d 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -145,7 +145,7 @@ org.scalatest scalatest_2.11 - 2.2.4 + ${scala.test.version} test diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml index 8e63fc69decad..d1c50b4c20096 100644 --- a/modules/scalar/pom.xml +++ b/modules/scalar/pom.xml @@ -72,7 +72,7 @@ org.scalatest scalatest_2.11 - 2.2.4 + ${scala.test.version} test diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml index e8d4c2f5acafd..b054c42a4e821 100644 --- a/modules/spark/pom.xml +++ b/modules/spark/pom.xml @@ -151,7 +151,7 @@ org.scalatest scalatest_2.11 - 2.2.6 + ${scala.test.version} test diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml index aad731964ad85..6975e739adc30 100644 --- a/modules/visor-console/pom.xml +++ b/modules/visor-console/pom.xml @@ -101,7 +101,7 @@ org.scalatest scalatest_2.11 - 2.2.4 + ${scala.test.version} test diff --git a/parent/pom.xml b/parent/pom.xml index e260118bcf7c0..4781d294f1922 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -107,6 +107,7 @@ 2.10.7 2.10.7 2.11.12 + 2.2.6 1.7.7 1.6.4 2.6.5 From 1607b7e5967953474e4deb55d2e49cc3f5fc120e Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Mon, 29 Oct 2018 15:36:46 +0700 Subject: [PATCH 133/403] IGNITE-9941 Web Console: Added configuration parameter to disable signup and possibility to create user accounts by administrator. (cherry picked from commit fe824a0e287f3b3432900357c3b09d939752ead7) --- modules/web-console/DEVNOTES.txt | 17 ++ modules/web-console/assembly/README.txt | 1 + modules/web-console/backend/app/settings.js | 4 +- .../backend/config/settings.json.sample | 6 +- modules/web-console/backend/routes/public.js | 22 +- modules/web-console/backend/services/mails.js | 24 +- modules/web-console/backend/services/users.js | 12 +- modules/web-console/frontend/app/app.js | 6 +- .../dialog-admin-create-user/component.ts | 27 +++ .../dialog-admin-create-user/controller.ts | 78 +++++++ .../dialog-admin-create-user/index.ts | 23 ++ .../dialog-admin-create-user/state.ts | 29 +++ .../dialog-admin-create-user/template.pug | 37 +++ .../app/components/form-signup/component.ts | 32 +++ .../app/components/form-signup/controller.ts | 46 ++++ .../types.ts => form-signup/index.ts} | 5 + .../app/components/form-signup/style.scss | 31 +++ .../app/components/form-signup/template.pug | 105 +++++++++ .../list-of-registered-users/column-defs.js | 23 -- .../list-of-registered-users/controller.js | 214 ++++++++++-------- .../list-of-registered-users/template.tpl.pug | 1 - .../app/components/page-admin/controller.ts | 28 +++ .../app/components/page-admin/index.js | 17 +- .../app/components/page-signup/component.js | 2 - .../{controller.js => controller.ts} | 48 ++-- .../app/components/page-signup/index.js | 3 +- .../app/components/page-signup/style.scss | 10 - .../app/components/page-signup/template.pug | 94 +------- .../user/{Auth.service.js => Auth.service.ts} | 90 ++++---- .../app/primitives/dropdown/index.pug | 3 +- .../app/primitives/form-field/checkbox.pug | 5 +- .../app/primitives/form-field/email.pug | 7 +- .../app/primitives/form-field/password.pug | 12 +- .../app/primitives/form-field/phone.pug | 5 +- .../frontend/app/utils/dialogState.ts | 56 +++++ 35 files changed, 776 insertions(+), 347 deletions(-) create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug create mode 100644 modules/web-console/frontend/app/components/form-signup/component.ts create mode 100644 modules/web-console/frontend/app/components/form-signup/controller.ts rename modules/web-console/frontend/app/components/{page-signup/types.ts => form-signup/index.ts} (89%) create mode 100644 modules/web-console/frontend/app/components/form-signup/style.scss create mode 100644 modules/web-console/frontend/app/components/form-signup/template.pug create mode 100644 modules/web-console/frontend/app/components/page-admin/controller.ts rename modules/web-console/frontend/app/components/page-signup/{controller.js => controller.ts} (53%) rename modules/web-console/frontend/app/modules/user/{Auth.service.js => Auth.service.ts} (51%) create mode 100644 modules/web-console/frontend/app/utils/dialogState.ts diff --git a/modules/web-console/DEVNOTES.txt b/modules/web-console/DEVNOTES.txt index ba61150b41bd9..a06608d14ad8a 100644 --- a/modules/web-console/DEVNOTES.txt +++ b/modules/web-console/DEVNOTES.txt @@ -139,3 +139,20 @@ Unit tests are performed with Mocha framework - https://mochajs.org To launch tests on your local machine you will need: 1. In new terminal change directory to 'modules/web-console/backend' folder and execute: "npm install". 2. To start test environment and tests execute: "npm run test". + + +Web Console settings +==================== +Web Console backend could be configured with custom parameters. + +See "backend/config/settings.json.sample" for list of parameters with example value. + +If you need custom parameters, you will need create "backend/config/settings.json" and adjust values. + +Web Console settings for Docker +=============================== +Web Console backend could be configured with custom parameters. + +You may pass custom parameters with help of "-e" option. + +For example: docker run -e "server_disable_signup=true" -p 9090:80 $IMAGE_ID diff --git a/modules/web-console/assembly/README.txt b/modules/web-console/assembly/README.txt index dd323993e5231..f1e114c523b0a 100644 --- a/modules/web-console/assembly/README.txt +++ b/modules/web-console/assembly/README.txt @@ -47,6 +47,7 @@ All available parameters with defaults: HTTPS key: --server:key "serve/keys/test.key" HTTPS certificate: --server:cert "serve/keys/test.crt" HTTPS passphrase: --server:keyPassphrase "password" + Disable self registration: --server:disable:signup false MongoDB URL: --mongodb:url mongodb://localhost/console Mail service: --mail:service "gmail" Signature text: --mail:sign "Kind regards, Apache Ignite Team" diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js index 104b66d690196..1c7504142095e 100644 --- a/modules/web-console/backend/app/settings.js +++ b/modules/web-console/backend/app/settings.js @@ -68,7 +68,9 @@ module.exports = { key: fs.readFileSync(nconf.get('server:key')), cert: fs.readFileSync(nconf.get('server:cert')), passphrase: nconf.get('server:keyPassphrase') - } + }, + // eslint-disable-next-line eqeqeq + disableSignup: nconf.get('server:disable:signup') == 'true' }, mail, mongoUrl: nconf.get('mongodb:url') || 'mongodb://127.0.0.1/console', diff --git a/modules/web-console/backend/config/settings.json.sample b/modules/web-console/backend/config/settings.json.sample index aa93e07c57eba..4070163c26b4b 100644 --- a/modules/web-console/backend/config/settings.json.sample +++ b/modules/web-console/backend/config/settings.json.sample @@ -5,8 +5,10 @@ "ssl": false, "key": "serve/keys/test.key", "cert": "serve/keys/test.crt", - "keyPassphrase": "password" - }, + "keyPassphrase": "password", + "disable": { + "signup": false + } }, "mongodb": { "url": "mongodb://localhost/console" }, diff --git a/modules/web-console/backend/routes/public.js b/modules/web-console/backend/routes/public.js index 155b2e9f268e4..8aa3b4338db4b 100644 --- a/modules/web-console/backend/routes/public.js +++ b/modules/web-console/backend/routes/public.js @@ -17,6 +17,7 @@ 'use strict'; +const _ = require('lodash'); const express = require('express'); const passport = require('passport'); @@ -49,15 +50,22 @@ module.exports.factory = function(mongo, mailsService, usersService, authService * Register new account. */ router.post('/signup', (req, res) => { - usersService.create(req.origin(), req.body) - .then((user) => new Promise((resolve, reject) => { - req.logIn(user, {}, (err) => { - if (err) - reject(err); + const createdByAdmin = _.get(req, 'user.admin', false); - resolve(user); + usersService.create(req.origin(), req.body, createdByAdmin) + .then((user) => { + if (createdByAdmin) + return user; + + return new Promise((resolve, reject) => { + req.logIn(user, {}, (err) => { + if (err) + reject(err); + + resolve(user); + }); }); - })) + }) .then(res.api.ok) .catch(res.api.error); }); diff --git a/modules/web-console/backend/services/mails.js b/modules/web-console/backend/services/mails.js index e0737cd2a7500..75da128ad9dca 100644 --- a/modules/web-console/backend/services/mails.js +++ b/modules/web-console/backend/services/mails.js @@ -81,17 +81,27 @@ module.exports.factory = (settings) => { } /** - * Send email to user for password reset. - * @param host - * @param user + * Send email when user signed up. + * + * @param host Web Console host. + * @param user User that signed up. + * @param createdByAdmin Whether user was created by admin. */ - emailUserSignUp(host, user) { + emailUserSignUp(host, user, createdByAdmin) { const resetLink = `${host}/password/reset?token=${user.resetPasswordToken}`; - return this.send(user, `Thanks for signing up for ${settings.mail.greeting}.`, + const sbj = createdByAdmin + ? 'Account was created for' + : 'Thanks for signing up for'; + + const reason = createdByAdmin + ? 'administrator created account for you' + : 'you have signed up'; + + return this.send(user, `${sbj} ${settings.mail.greeting}.`, `Hello ${user.firstName} ${user.lastName}!

` + - `You are receiving this email because you have signed up to use ${settings.mail.greeting}.

` + - 'If you have not done the sign up and do not know what this email is about, please ignore it.
' + + `You are receiving this email because ${reason} to use ${settings.mail.greeting}.

` + + 'If you do not know what this email is about, please ignore it.
' + 'You may reset the password by clicking on the following link, or paste this into your browser:

' + `${resetLink}`); } diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js index 43bceda3cd6d5..79578c5b45995 100644 --- a/modules/web-console/backend/services/users.js +++ b/modules/web-console/backend/services/users.js @@ -42,17 +42,21 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService, /** * Save profile information. * - * @param {String} host - The host - * @param {Object} user - The user + * @param {String} host - The host. + * @param {Object} user - The user. + * @param {Object} createdByAdmin - Whether user created by admin. * @returns {Promise.} that resolves account id of merge operation. */ - static create(host, user) { + static create(host, user, createdByAdmin) { return mongo.Account.count().exec() .then((cnt) => { user.admin = cnt === 0; user.registered = new Date(); user.token = utilsService.randomString(settings.tokenLength); + if (settings.server.disableSignup && !user.admin && !createdByAdmin) + throw new errors.ServerErrorException('Sign-up is not allowed. Ask your Web Console administrator to create account for you.'); + return new mongo.Account(user); }) .then((created) => { @@ -74,7 +78,7 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService, return registered.save() .then(() => mongo.Space.create({name: 'Personal space', owner: registered._id})) .then(() => { - mailsService.emailUserSignUp(host, registered); + mailsService.emailUserSignUp(host, registered, createdByAdmin); return registered; }); diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index c890fa42db588..47374aa31fbca 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -112,6 +112,7 @@ import id8 from './filters/id8.filter'; // Components import igniteListOfRegisteredUsers from './components/list-of-registered-users'; +import dialogAdminCreateUser from './components/dialog-admin-create-user'; import IgniteActivitiesUserDialog from './components/activities-user-dialog'; import './components/input-dialog'; import webConsoleHeader from './components/web-console-header'; @@ -156,6 +157,7 @@ import pagePasswordReset from './components/page-password-reset'; import pageSignup from './components/page-signup'; import pageSignin from './components/page-signin'; import pageForgotPassword from './components/page-forgot-password'; +import formSignup from './components/form-signup'; import igniteServices from './services'; @@ -245,6 +247,7 @@ export default angular.module('ignite-console', [ connectedClusters.name, connectedClustersDialog.name, igniteListOfRegisteredUsers.name, + dialogAdminCreateUser.name, pageProfile.name, pageLanding.name, pagePasswordChanged.name, @@ -260,7 +263,8 @@ export default angular.module('ignite-console', [ igniteChartSelector.name, statusOutput.name, progressLine.name, - formField.name + formField.name, + formSignup.name ]) .service('$exceptionHandler', $exceptionHandler) // Directives. diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts new file mode 100644 index 0000000000000..3a60d93077030 --- /dev/null +++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import {DialogAdminCreateUser} from './controller'; + +export default { + template, + controller: DialogAdminCreateUser, + bindings: { + close: '&onHide' + } +}; diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts new file mode 100644 index 0000000000000..e8a29e14212ad --- /dev/null +++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Auth from '../../modules/user/Auth.service'; +import MessagesFactory from '../../services/Messages.service'; +import FormUtilsFactoryFactory from '../../services/FormUtils.service'; +import LoadingServiceFactory from '../../modules/loading/loading.service'; +import {ISignupData} from '../form-signup'; + +export class DialogAdminCreateUser { + close: ng.ICompiledExpression; + + form: ng.IFormController; + + data: ISignupData = { + email: null, + password: null, + firstName: null, + lastName: null, + company: null, + country: null + }; + + serverError: string | null = null; + + static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils', 'IgniteLoading']; + + constructor( + private Auth: Auth, + private IgniteMessages: ReturnType, + private IgniteFormUtils: ReturnType, + private loading: ReturnType + ) {} + + canSubmitForm(form: DialogAdminCreateUser['form']) { + return form.$error.server ? true : !form.$invalid; + } + + setServerError(error: DialogAdminCreateUser['serverError']) { + this.serverError = error; + } + + createUser() { + this.IgniteFormUtils.triggerValidation(this.form); + + this.setServerError(null); + + if (!this.canSubmitForm(this.form)) + return; + + this.loading.start('createUser'); + + this.Auth.signup(this.data, false) + .then(() => { + this.IgniteMessages.showInfo(`User ${this.data.email} created`); + this.close({}); + }) + .catch((res) => { + this.loading.finish('createUser'); + this.IgniteMessages.showError(null, res.data); + this.setServerError(res.data); + }); + } +} diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts new file mode 100644 index 0000000000000..5a23368229030 --- /dev/null +++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import component from './component'; +import {registerState} from './state'; + +export default angular.module('ignite-console.dialog-admin-create-user', []) + .run(registerState) + .component('dialogAdminCreateUser', component); diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts new file mode 100644 index 0000000000000..c64238e6c882d --- /dev/null +++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {UIRouter} from '@uirouter/angularjs'; +import {dialogState} from '../../utils/dialogState'; + +registerState.$inject = ['$uiRouter']; + +export function registerState(router: UIRouter) { + router.stateRegistry.register({ + ...dialogState('dialog-admin-create-user'), + name: 'base.settings.admin.createUser', + url: '/create-user' + }); +} diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug b/modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug new file mode 100644 index 0000000000000..0a9f2b4c4b67c --- /dev/null +++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug @@ -0,0 +1,37 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.modal-dialog(ng-click='$event.stopPropagation()' ) + form.modal-content( + name='$ctrl.form' novalidate + ignite-loading='createUser' + ignite-loading-text='Creating user…' + ) + .modal-header + h4.modal-title Create User + + button.close(type='button' aria-label='Close' ng-click='$ctrl.close()') + svg(ignite-icon="cross") + .modal-body + form-signup( + outer-form='$ctrl.form' + ng-model='$ctrl.data' + server-error='$ctrl.serverError' + ) + .modal-footer + div + button.btn-ignite.btn-ignite--link-success(ng-click='$ctrl.close()') Cancel + button.btn-ignite.btn-ignite--success(ng-click='$ctrl.createUser()') Create user diff --git a/modules/web-console/frontend/app/components/form-signup/component.ts b/modules/web-console/frontend/app/components/form-signup/component.ts new file mode 100644 index 0000000000000..79863db09c84b --- /dev/null +++ b/modules/web-console/frontend/app/components/form-signup/component.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import './style.scss'; +import {FormSignup} from './controller'; + +export const component: ng.IComponentOptions = { + template, + controller: FormSignup, + bindings: { + outerForm: '<', + serverError: '<' + }, + require: { + ngModel: 'ngModel' + } +}; diff --git a/modules/web-console/frontend/app/components/form-signup/controller.ts b/modules/web-console/frontend/app/components/form-signup/controller.ts new file mode 100644 index 0000000000000..eedf6d0ca0b07 --- /dev/null +++ b/modules/web-console/frontend/app/components/form-signup/controller.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import CountriesService from '../../services/Countries.service'; +import {ISignupFormController} from '.'; + +export class FormSignup implements ng.IPostLink, ng.IOnDestroy, ng.IOnChanges { + static $inject = ['IgniteCountries']; + + constructor(private Countries: ReturnType) {} + + countries = this.Countries.getAll(); + + innerForm: ISignupFormController; + outerForm: ng.IFormController; + ngModel: ng.INgModelController; + serverError: string | null = null; + + $postLink() { + this.outerForm.$addControl(this.innerForm); + this.innerForm.email.$validators.server = () => !this.serverError; + } + + $onDestroy() { + this.outerForm.$removeControl(this.innerForm); + } + + $onChanges(changes: {serverError: ng.IChangesObject}) { + if (changes.serverError && this.innerForm) + this.innerForm.email.$validate(); + } +} diff --git a/modules/web-console/frontend/app/components/page-signup/types.ts b/modules/web-console/frontend/app/components/form-signup/index.ts similarity index 89% rename from modules/web-console/frontend/app/components/page-signup/types.ts rename to modules/web-console/frontend/app/components/form-signup/index.ts index 49e8d5d15072b..1798aa0ae0ea6 100644 --- a/modules/web-console/frontend/app/components/page-signup/types.ts +++ b/modules/web-console/frontend/app/components/form-signup/index.ts @@ -15,6 +15,11 @@ * limitations under the License. */ +import {component} from './component'; + +export default angular.module('ignite-console.form-signup', []) + .component('formSignup', component); + export interface ISignupData { email: string, password: string, diff --git a/modules/web-console/frontend/app/components/form-signup/style.scss b/modules/web-console/frontend/app/components/form-signup/style.scss new file mode 100644 index 0000000000000..e80b5108131ef --- /dev/null +++ b/modules/web-console/frontend/app/components/form-signup/style.scss @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +form-signup { + .form-signup__grid { + display: grid; + grid-gap: 10px; + grid-template-columns: 1fr 1fr; + + .span-1 { + grid-column: span 1; + } + .span-2 { + grid-column: span 2; + } + } +} diff --git a/modules/web-console/frontend/app/components/form-signup/template.pug b/modules/web-console/frontend/app/components/form-signup/template.pug new file mode 100644 index 0000000000000..375ea958e3d18 --- /dev/null +++ b/modules/web-console/frontend/app/components/form-signup/template.pug @@ -0,0 +1,105 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +include /app/helpers/jade/mixins + +.form-signup__grid(ng-form='signup' ng-ref='$ctrl.innerForm' ng-ref-read='form') + .span-2 + +form-field__email({ + label: 'Email:', + model: '$ctrl.ngModel.$viewValue.email', + name: '"email"', + placeholder: 'Input email', + required: true + })( + ng-model-options='{allowInvalid: true}' + autocomplete='email' + ignite-auto-focus + ) + +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) + .span-1 + +form-field__password({ + label: 'Password:', + model: '$ctrl.ngModel.$viewValue.password', + name: '"password"', + placeholder: 'Input password', + required: true + })( + autocomplete='new-password' + ) + .span-1 + +form-field__password({ + label: 'Confirm:', + model: 'confirm', + name: '"confirm"', + placeholder: 'Confirm password', + required: true + })( + ignite-match='$ctrl.ngModel.$viewValue.password' + autocomplete='off' + ) + .span-1 + +form-field__text({ + label: 'First name:', + model: '$ctrl.ngModel.$viewValue.firstName', + name: '"firstName"', + placeholder: 'Input first name', + required: true + })( + autocomplete='given-name' + ) + .span-1 + +form-field__text({ + label: 'Last name:', + model: '$ctrl.ngModel.$viewValue.lastName', + name: '"lastName"', + placeholder: 'Input last name', + required: true + })( + autocomplete='family-name' + ) + .span-1 + +form-field__phone({ + label: 'Phone:', + model: '$ctrl.ngModel.$viewValue.phone', + name: '"phone"', + placeholder: 'Input phone (ex.: +15417543010)', + optional: true + })( + autocomplete='tel' + ) + .span-1 + +form-field__dropdown({ + label: 'Country:', + model: '$ctrl.ngModel.$viewValue.country', + name: '"country"', + required: true, + placeholder: 'Choose your country', + options: '$ctrl.countries' + })( + autocomplete='country' + ) + .span-2 + +form-field__text({ + label: 'Company:', + model: '$ctrl.ngModel.$viewValue.company', + name: '"company"', + placeholder: 'Input company name', + required: true + })( + ignite-on-enter-focus-move='countryInput' + autocomplete='organization' + ) diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js index 419a0636f23aa..2094f0c7909c7 100644 --- a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js +++ b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js @@ -25,34 +25,11 @@ const MODEL_HEADER_TEMPLATE = `
${ICON_SORT}
`; const IGFS_HEADER_TEMPLATE = `
${ICON_SORT}
`; -const ACTIONS_TEMPLATE = ` -`; - const EMAIL_TEMPLATE = ''; const DATE_WITH_TITLE = '
'; const VALUE_WITH_TITLE = '
'; export default [ - {name: 'actions', enableHiding: false, displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, visible: false}, {name: 'user', enableHiding: false, displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, pinnedLeft: true, filter: { placeholder: 'Filter by name...' }}, {name: 'email', displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, width: 220, enableFiltering: true, filter: { placeholder: 'Filter by email...' }}, {name: 'company', displayName: 'Company', categoryDisplayName: 'Company', field: 'company', cellTemplate: VALUE_WITH_TITLE, minWidth: 180, enableFiltering: true, filter: { placeholder: 'Filter by company...' }}, diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/controller.js index f8b2797359f52..6a7e098df0f55 100644 --- a/modules/web-console/frontend/app/components/list-of-registered-users/controller.js +++ b/modules/web-console/frontend/app/components/list-of-registered-users/controller.js @@ -15,11 +15,13 @@ * limitations under the License. */ -import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug'; +import _ from 'lodash'; import columnDefs from './column-defs'; import categories from './categories'; +import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug'; + const rowTemplate = `
$ctrl.user = user); - - const becomeUser = () => { - const user = this.gridApi.selection.legacyGetSelectedRows()[0]; - - AdminData.becomeUser(user._id) - .then(() => User.load()) - .then(() => $state.go('default-state')) - .then(() => NotebookData.load()); - }; - - const removeUser = () => { - const user = this.gridApi.selection.legacyGetSelectedRows()[0]; - - Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`) - .then(() => AdminData.removeUser(user)) - .then(() => { - const i = _.findIndex($ctrl.gridOptions.data, (u) => u._id === user._id); + this.uiGridPinningConstants = uiGridPinningConstants; + this.uiGridGroupingConstants = uiGridGroupingConstants; - if (i >= 0) { - $ctrl.gridOptions.data.splice(i, 1); - $ctrl.gridApi.selection.clearSelectedRows(); - } - - $ctrl.adjustHeight($ctrl.gridOptions.data.length); - - return $ctrl._refreshRows(); - }); - }; - - const toggleAdmin = () => { - const user = this.gridApi.selection.legacyGetSelectedRows()[0]; - - if (user.adminChanging) - return; - - user.adminChanging = true; - - AdminData.toggleAdmin(user) - .finally(() => { - $ctrl._updateSelected(); - - user.adminChanging = false; - }); - }; - - const showActivities = () => { - const user = this.gridApi.selection.legacyGetSelectedRows()[0]; - - return new ActivitiesUserDialog({ user }); - }; + User.read().then((user) => this.user = user); const companiesExcludeFilter = (renderableRows) => { - if (_.isNil($ctrl.params.companiesExclude)) + if (_.isNil(this.params.companiesExclude)) return renderableRows; _.forEach(renderableRows, (row) => { - row.visible = _.isEmpty($ctrl.params.companiesExclude) || - row.entity.company.toLowerCase().indexOf($ctrl.params.companiesExclude.toLowerCase()) === -1; + row.visible = _.isEmpty(this.params.companiesExclude) || + row.entity.company.toLowerCase().indexOf(this.params.companiesExclude.toLowerCase()) === -1; }); return renderableRows; }; - $ctrl.actionOptions = [ + this.actionOptions = [ { action: 'Become this user', - click: becomeUser.bind(this), + click: () => this.becomeUser(), available: true }, { action: 'Revoke admin', - click: toggleAdmin.bind(this), + click: () => this.toggleAdmin(), available: true }, { action: 'Grant admin', - click: toggleAdmin.bind(this), + click: () => this.toggleAdmin(), available: false }, + { + action: 'Add user', + sref: '.createUser', + available: true + }, { action: 'Remove user', - click: removeUser.bind(this), + click: () => this.removeUser(), available: true }, { action: 'Activity detail', - click: showActivities.bind(this), + click: () => this.showActivities(), available: true } ]; - $ctrl._userGridOptions = { + this._userGridOptions = { columnDefs, categories }; - $ctrl.gridOptions = { + this.gridOptions = { data: [], columnDefs, @@ -176,18 +138,18 @@ export default class IgniteListOfRegisteredUsersCtrl { rowIdentity: (row) => row._id, getRowIdentity: (row) => row._id, onRegisterApi: (api) => { - $ctrl.gridApi = api; + this.gridApi = api; - api.selection.on.rowSelectionChanged($scope, $ctrl._updateSelected.bind($ctrl)); - api.selection.on.rowSelectionChangedBatch($scope, $ctrl._updateSelected.bind($ctrl)); + api.selection.on.rowSelectionChanged($scope, this._updateSelected.bind(this)); + api.selection.on.rowSelectionChangedBatch($scope, this._updateSelected.bind(this)); - api.core.on.filterChanged($scope, $ctrl._filteredRows.bind($ctrl)); - api.core.on.rowsVisibleChanged($scope, $ctrl._filteredRows.bind($ctrl)); + api.core.on.filterChanged($scope, this._filteredRows.bind(this)); + api.core.on.rowsVisibleChanged($scope, this._filteredRows.bind(this)); api.grid.registerRowsProcessor(companiesExcludeFilter, 50); - $scope.$watch(() => $ctrl.gridApi.grid.getVisibleRows().length, (rows) => $ctrl.adjustHeight(rows)); - $scope.$watch(() => $ctrl.params.companiesExclude, () => $ctrl.gridApi.grid.refreshRows()); + $scope.$watch(() => this.gridApi.grid.getVisibleRows().length, (rows) => this.adjustHeight(rows)); + $scope.$watch(() => this.params.companiesExclude, () => this.gridApi.grid.refreshRows()); } }; @@ -197,20 +159,20 @@ export default class IgniteListOfRegisteredUsersCtrl { const reloadUsers = (params) => { AdminData.loadUsers(params) .then((data) => { - $ctrl.gridOptions.data = data; + this.gridOptions.data = data; - $ctrl.companies = _.values(_.groupBy(data, 'company')); - $ctrl.countries = _.values(_.groupBy(data, 'countryCode')); + this.companies = _.values(_.groupBy(data, 'company')); + this.countries = _.values(_.groupBy(data, 'countryCode')); - $ctrl._refreshRows(); + this._refreshRows(); }); }; const filterDates = _.debounce(() => { - const sdt = $ctrl.params.startDate; - const edt = $ctrl.params.endDate; + const sdt = this.params.startDate; + const edt = this.params.endDate; - $ctrl.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`; + this.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`; const startDate = Date.UTC(sdt.getFullYear(), sdt.getMonth(), 1); const endDate = Date.UTC(edt.getFullYear(), edt.getMonth() + 1, 1); @@ -218,8 +180,9 @@ export default class IgniteListOfRegisteredUsersCtrl { reloadUsers({ startDate, endDate }); }, 250); - $scope.$watch(() => $ctrl.params.startDate, filterDates); - $scope.$watch(() => $ctrl.params.endDate, filterDates); + $scope.$on('userCreated', filterDates); + $scope.$watch(() => this.params.startDate, filterDates); + $scope.$watch(() => this.params.endDate, filterDates); } adjustHeight(rows) { @@ -233,36 +196,95 @@ export default class IgniteListOfRegisteredUsersCtrl { _filteredRows() { const filtered = _.filter(this.gridApi.grid.rows, ({ visible}) => visible); - const entities = _.map(filtered, 'entity'); - this.filteredRows = entities; + this.filteredRows = _.map(filtered, 'entity'); } _updateSelected() { const ids = this.gridApi.selection.legacyGetSelectedRows().map(({ _id }) => _id).sort(); + if (!_.isEqual(ids, this.selected)) + this.selected = ids; + if (ids.length) { const user = this.gridApi.selection.legacyGetSelectedRows()[0]; const other = this.user._id !== user._id; - this.actionOptions[1].available = other && user.admin; - this.actionOptions[2].available = other && !user.admin; - - this.actionOptions[0].available = other; - this.actionOptions[3].available = other; + this.actionOptions[0].available = other; // Become this user. + this.actionOptions[1].available = other && user.admin; // Revoke admin. + this.actionOptions[2].available = other && !user.admin; // Grant admin. + this.actionOptions[4].available = other; // Remove user. + this.actionOptions[5].available = true; // Activity detail. + } + else { + this.actionOptions[0].available = false; // Become this user. + this.actionOptions[1].available = false; // Revoke admin. + this.actionOptions[2].available = false; // Grant admin. + this.actionOptions[4].available = false; // Remove user. + this.actionOptions[5].available = false; // Activity detail. } - - if (!_.isEqual(ids, this.selected)) - this.selected = ids; } _refreshRows() { if (this.gridApi) { this.gridApi.grid.refreshRows() - .then(() => this.selected.length && this._updateSelected()); + .then(() => this._updateSelected()); } } + becomeUser() { + const user = this.gridApi.selection.legacyGetSelectedRows()[0]; + + this.AdminData.becomeUser(user._id) + .then(() => this.User.load()) + .then(() => this.$state.go('default-state')) + .then(() => this.NotebookData.load()); + } + + toggleAdmin() { + if (!this.gridApi) + return; + + const user = this.gridApi.selection.legacyGetSelectedRows()[0]; + + if (user.adminChanging) + return; + + user.adminChanging = true; + + this.AdminData.toggleAdmin(user) + .finally(() => { + this._updateSelected(); + + user.adminChanging = false; + }); + } + + removeUser() { + const user = this.gridApi.selection.legacyGetSelectedRows()[0]; + + this.Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`) + .then(() => this.AdminData.removeUser(user)) + .then(() => { + const i = _.findIndex(this.gridOptions.data, (u) => u._id === user._id); + + if (i >= 0) { + this.gridOptions.data.splice(i, 1); + this.gridApi.selection.clearSelectedRows(); + } + + this.adjustHeight(this.gridOptions.data.length); + + return this._refreshRows(); + }); + } + + showActivities() { + const user = this.gridApi.selection.legacyGetSelectedRows()[0]; + + return new this.ActivitiesDialogFactory({ user }); + } + groupByUser() { this.groupBy = 'user'; diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug index d32b64d082f12..ff1c851817f65 100644 --- a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug +++ b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug @@ -74,7 +74,6 @@ ul.tabs.tabs--blue label: 'Actions', model: '$ctrl.action', name: 'action', - disabled: '!$ctrl.selected.length', options: '$ctrl.actionOptions' }) diff --git a/modules/web-console/frontend/app/components/page-admin/controller.ts b/modules/web-console/frontend/app/components/page-admin/controller.ts new file mode 100644 index 0000000000000..45a5c0a5547e8 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-admin/controller.ts @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import UserNotificationsService from '../user-notifications/service'; + +export default class PageAdminCtrl { + static $inject = ['UserNotifications']; + + constructor(private UserNotifications: UserNotificationsService) {} + + changeUserNotifications() { + this.UserNotifications.editor(); + } +} diff --git a/modules/web-console/frontend/app/components/page-admin/index.js b/modules/web-console/frontend/app/components/page-admin/index.js index 232d889eb46e1..8b26f2c410290 100644 --- a/modules/web-console/frontend/app/components/page-admin/index.js +++ b/modules/web-console/frontend/app/components/page-admin/index.js @@ -17,23 +17,12 @@ import './style.scss'; +import controller from './controller'; import templateUrl from './template.tpl.pug'; -class PageAdminCtrl { - static $inject = ['UserNotifications']; - - constructor(UserNotifications) { - this.UserNotifications = UserNotifications; - } - - changeUserNotifications() { - this.UserNotifications.editor(); - } -} - export default angular .module('ignite-console.page-admin', []) .component('pageAdmin', { - templateUrl, - controller: PageAdminCtrl + controller, + templateUrl }); diff --git a/modules/web-console/frontend/app/components/page-signup/component.js b/modules/web-console/frontend/app/components/page-signup/component.js index 789a68187e828..968ff396e298d 100644 --- a/modules/web-console/frontend/app/components/page-signup/component.js +++ b/modules/web-console/frontend/app/components/page-signup/component.js @@ -15,8 +15,6 @@ * limitations under the License. */ -import angular from 'angular'; - import template from './template.pug'; import controller from './controller'; import './style.scss'; diff --git a/modules/web-console/frontend/app/components/page-signup/controller.js b/modules/web-console/frontend/app/components/page-signup/controller.ts similarity index 53% rename from modules/web-console/frontend/app/components/page-signup/controller.js rename to modules/web-console/frontend/app/components/page-signup/controller.ts index 10397760c3d71..ddf77be099ca9 100644 --- a/modules/web-console/frontend/app/components/page-signup/controller.js +++ b/modules/web-console/frontend/app/components/page-signup/controller.ts @@ -15,11 +15,15 @@ * limitations under the License. */ +import Auth from '../../modules/user/Auth.service'; +import MessagesFactory from '../../services/Messages.service'; +import FormUtilsFactoryFactory from '../../services/FormUtils.service'; +import {ISignupData} from '../form-signup'; + export default class PageSignup { - /** @type {import('./types').ISignupFormController} */ - form; - /** @type {import('./types').ISignupData} */ - data = { + form: ng.IFormController; + + data: ISignupData = { email: null, password: null, firstName: null, @@ -27,37 +31,23 @@ export default class PageSignup { company: null, country: null }; - /** @type {string} */ - serverError = null; - static $inject = ['IgniteCountries', 'Auth', 'IgniteMessages', 'IgniteFormUtils']; + serverError: string | null = null; - /** - * @param {ReturnType} Countries - * @param {import('app/modules/user/Auth.service').default} Auth - * @param {ReturnType} IgniteMessages - * @param {ReturnType} IgniteFormUtils - */ - constructor(Countries, Auth, IgniteMessages, IgniteFormUtils) { - this.Auth = Auth; - this.IgniteMessages = IgniteMessages; - this.countries = Countries.getAll(); - this.IgniteFormUtils = IgniteFormUtils; - } + static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils']; - /** @param {import('./types').ISignupFormController} form */ - canSubmitForm(form) { - return form.$error.server ? true : !form.$invalid; - } + constructor( + private Auth: Auth, + private IgniteMessages: ReturnType, + private IgniteFormUtils: ReturnType + ) {} - $postLink() { - this.form.email.$validators.server = () => !this.serverError; + canSubmitForm(form: PageSignup['form']) { + return form.$error.server ? true : !form.$invalid; } - /** @param {string} error */ - setServerError(error) { + setServerError(error: PageSignup['serverError']) { this.serverError = error; - this.form.email.$validate(); } signup() { @@ -68,7 +58,7 @@ export default class PageSignup { if (!this.canSubmitForm(this.form)) return; - return this.Auth.signnup(this.data).catch((res) => { + return this.Auth.signup(this.data).catch((res) => { this.IgniteMessages.showError(null, res.data); this.setServerError(res.data); }); diff --git a/modules/web-console/frontend/app/components/page-signup/index.js b/modules/web-console/frontend/app/components/page-signup/index.js index 4efadb5f47af5..001d26987d29a 100644 --- a/modules/web-console/frontend/app/components/page-signup/index.js +++ b/modules/web-console/frontend/app/components/page-signup/index.js @@ -22,7 +22,8 @@ import {registerState} from './run'; export default angular .module('ignite-console.page-signup', [ 'ui.router', - 'ignite-console.user' + 'ignite-console.user', + 'ignite-console.form-signup' ]) .component('pageSignup', component) .run(registerState); diff --git a/modules/web-console/frontend/app/components/page-signup/style.scss b/modules/web-console/frontend/app/components/page-signup/style.scss index 93167ce6856da..98c4729bf5ad5 100644 --- a/modules/web-console/frontend/app/components/page-signup/style.scss +++ b/modules/web-console/frontend/app/components/page-signup/style.scss @@ -41,16 +41,6 @@ page-signup { margin-left: auto; } } - - form { - display: grid; - grid-gap: 10px; - grid-template-columns: 1fr 1fr; - - .full-width { - grid-column: 1 / 3; - } - } } .page-signup__has-account-message { diff --git a/modules/web-console/frontend/app/components/page-signup/template.pug b/modules/web-console/frontend/app/components/page-signup/template.pug index 11bb50a15c733..5584dfcb67129 100644 --- a/modules/web-console/frontend/app/components/page-signup/template.pug +++ b/modules/web-console/frontend/app/components/page-signup/template.pug @@ -22,95 +22,13 @@ web-console-header .container--responsive.body-container section - -var form = '$ctrl.form' h3 Don't Have An Account? - form(name=form novalidate ng-submit='$ctrl.signup()') - .full-width - +form-field__email({ - label: 'Email:', - model: '$ctrl.data.email', - name: '"email"', - placeholder: 'Input email', - required: true - })( - ng-model-options='{allowInvalid: true}' - autocomplete='email' - ignite-auto-focus - ) - +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) - div - +form-field__password({ - label: 'Password:', - model: '$ctrl.data.password', - name: '"password"', - placeholder: 'Input password', - required: true - })( - autocomplete='new-password' - ) - div - +form-field__password({ - label: 'Confirm:', - model: 'confirm', - name: '"confirm"', - placeholder: 'Confirm password', - required: true - })( - ignite-match='$ctrl.data.password' - autocomplete='off' - ) - div - +form-field__text({ - label: 'First name:', - model: '$ctrl.data.firstName', - name: '"firstName"', - placeholder: 'Input first name', - required: true - })( - autocomplete='given-name' - ) - div - +form-field__text({ - label: 'Last name:', - model: '$ctrl.data.lastName', - name: '"lastName"', - placeholder: 'Input last name', - required: true - })( - autocomplete='family-name' - ) - div - +form-field__phone({ - label: 'Phone:', - model: '$ctrl.data.phone', - name: '"phone"', - placeholder: 'Input phone (ex.: +15417543010)', - optional: true - })( - autocomplete='tel' - ) - div - +form-field__dropdown({ - label: 'Country:', - model: '$ctrl.data.country', - name: '"country"', - required: true, - placeholder: 'Choose your country', - options: '$ctrl.countries' - })( - autocomplete='country' - ) - .full-width - +form-field__text({ - label: 'Company:', - model: '$ctrl.data.company', - name: '"company"', - placeholder: 'Input company name', - required: true - })( - ignite-on-enter-focus-move='countryInput' - autocomplete='organization' - ) + form(name='$ctrl.form' novalidate ng-submit='$ctrl.signup()') + form-signup( + outer-form='$ctrl.form' + ng-model='$ctrl.data' + server-error='$ctrl.serverError' + ) footer.full-width.form-footer button.btn-ignite.btn-ignite--primary( type='submit' diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.ts similarity index 51% rename from modules/web-console/frontend/app/modules/user/Auth.service.js rename to modules/web-console/frontend/app/modules/user/Auth.service.ts index 8c3cc4ea4e2db..55956ad990b47 100644 --- a/modules/web-console/frontend/app/modules/user/Auth.service.js +++ b/modules/web-console/frontend/app/modules/user/Auth.service.ts @@ -15,64 +15,53 @@ * limitations under the License. */ -/** - * @typedef {object} SignupUserInfo - * @prop {string} email - * @prop {string} password - * @prop {string} firstName - * @prop {string} lastName - * @prop {string} company - * @prop {string} country - */ +import {StateService} from '@uirouter/angularjs'; +import MessagesFactory from '../../services/Messages.service'; +import {service as GettingsStartedFactory} from '../../modules/getting-started/GettingStarted.provider'; +import UserServiceFactory from './User.service'; + +type SignupUserInfo = { + email: string, + password: string, + firstName: string, + lastName: string, + company: string, + country: string, +}; + +type AuthActions = 'signin' | 'signup' | 'password/forgot'; +type AuthOptions = {email:string, password:string}|SignupUserInfo|{email:string}; export default class AuthService { static $inject = ['$http', '$rootScope', '$state', '$window', 'IgniteMessages', 'gettingStarted', 'User']; - /** - * @param {ng.IHttpService} $http - * @param {ng.IRootScopeService} $root - * @param {import('@uirouter/angularjs').StateService} $state - * @param {ng.IWindowService} $window - * @param {ReturnType} Messages - * @param {ReturnType} gettingStarted - * @param {ReturnType} User - */ - constructor($http, $root, $state, $window, Messages, gettingStarted, User) { - this.$http = $http; - this.$root = $root; - this.$state = $state; - this.$window = $window; - this.Messages = Messages; - this.gettingStarted = gettingStarted; - this.User = User; - } - /** - * @param {SignupUserInfo} userInfo - */ - signnup(userInfo) { - return this._auth('signup', userInfo); + + constructor( + private $http: ng.IHttpService, + private $root: ng.IRootScopeService, + private $state: StateService, + private $window: ng.IWindowService, + private Messages: ReturnType, + private gettingStarted: ReturnType, + private User: ReturnType + ) {} + + signup(userInfo: SignupUserInfo, loginAfterSignup: boolean = true) { + return this._auth('signup', userInfo, loginAfterSignup); } - /** - * @param {string} email - * @param {string} password - */ - signin(email, password) { + + signin(email: string, password: string) { return this._auth('signin', {email, password}); } - /** - * @param {string} email - */ - remindPassword(email) { + + remindPassword(email: string) { return this._auth('password/forgot', {email}).then(() => this.$state.go('password.send')); } // TODO IGNITE-7994: Remove _auth and move API calls to corresponding methods /** * Performs the REST API call. - * @private - * @param {('signin'|'signup'|'password/forgot')} action - * @param {{email:string,password:string}|SignupUserInfo|{email:string}} userInfo */ - _auth(action, userInfo) { + private _auth(action: AuthActions, userInfo: AuthOptions, loginAfterwards: boolean = true) { return this.$http.post('/api/v1/' + action, userInfo) .then(() => { if (action === 'password/forgot') @@ -80,11 +69,12 @@ export default class AuthService { this.User.read() .then((user) => { - this.$root.$broadcast('user', user); - - this.$state.go('default-state'); - - this.$root.gettingStarted.tryShow(); + if (loginAfterwards) { + this.$root.$broadcast('user', user); + this.$state.go('default-state'); + this.$root.gettingStarted.tryShow(); + } else + this.$root.$broadcast('userCreated'); }); }); } diff --git a/modules/web-console/frontend/app/primitives/dropdown/index.pug b/modules/web-console/frontend/app/primitives/dropdown/index.pug index 0099457be59ab..aad6efd003605 100644 --- a/modules/web-console/frontend/app/primitives/dropdown/index.pug +++ b/modules/web-console/frontend/app/primitives/dropdown/index.pug @@ -37,5 +37,6 @@ mixin ignite-form-field-bsdropdown({label, model, name, disabled, required, opti ul.dropdown-menu(role='menu') li(ng-repeat=`item in ${options}` ng-if='item.available') - a(ng-click='item.click()') {{ item.action }} + a(ng-if='item.click' ng-click='item.click()') {{ item.action }} + a(ng-if='item.sref' ui-sref='{{:: item.sref}}') {{ item.action }} diff --git a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug index 88b8f5a2f9618..fe0f808574b3b 100644 --- a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug +++ b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug @@ -15,7 +15,7 @@ limitations under the License. mixin form-field__checkbox({ label, model, name, disabled, required, tip, tipOpts }) - .form-field.form-field__checkbox(id=`{{ ${name} }}Field`) + .form-field.ignite-form-field.form-field__checkbox(id=`{{ ${name} }}Field`) +form-field__label({ label, name, required, disabled }) +form-field__tooltip({ title: tip, options: tipOpts }) @@ -24,7 +24,8 @@ mixin form-field__checkbox({ label, model, name, disabled, required, tip, tipOpt +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes) .form-field__errors( - ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}` + ng-messages=`$input.$error` + ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) if block block diff --git a/modules/web-console/frontend/app/primitives/form-field/email.pug b/modules/web-console/frontend/app/primitives/form-field/email.pug index b68a520811741..5fb0c3019fc67 100644 --- a/modules/web-console/frontend/app/primitives/form-field/email.pug +++ b/modules/web-console/frontend/app/primitives/form-field/email.pug @@ -15,9 +15,9 @@ limitations under the License. mixin form-field__email({ label, model, name, disabled, required, placeholder, tip }) - -var errLbl = label.substring(0, label.length - 1) + -let errLbl = label[label.length - 1] === ':' ? label.substring(0, label.length - 1) : label - .form-field + .form-field.ignite-form-field +form-field__label({ label, name, required, disabled }) +form-field__tooltip({ title: tip, options: tipOpts }) @@ -26,7 +26,8 @@ mixin form-field__email({ label, model, name, disabled, required, placeholder, t +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes) .form-field__errors( - ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}` + ng-messages=`$input.$error` + ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) if block block diff --git a/modules/web-console/frontend/app/primitives/form-field/password.pug b/modules/web-console/frontend/app/primitives/form-field/password.pug index 6b9818bc91f43..374d768213639 100644 --- a/modules/web-console/frontend/app/primitives/form-field/password.pug +++ b/modules/web-console/frontend/app/primitives/form-field/password.pug @@ -17,9 +17,9 @@ mixin form-field__password({ label, model, name, disabled, required, placeholder, tip }) -var errLbl = label.substring(0, label.length - 1) - .form-field.form-field__password( + .form-field.form-field__password.ignite-form-field( password-visibility-root - on-password-visibility-toggle=`${form}[${name}].$setTouched()` + on-password-visibility-toggle=`$input1.$setTouched(); $input2.$setTouched()` ) +form-field__label({ label, name, required }) +form-field__tooltip({ title: tip, options: tipOpts }) @@ -27,16 +27,22 @@ mixin form-field__password({ label, model, name, disabled, required, placeholder .form-field__control - attributes.type='password' - attributes.class = 'password-visibility__password-hidden' + - attributes['ng-ref'] = '$input1' +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes) - attributes.class = 'password-visibility__password-visible' - attributes.type='text' - attributes.autocomplete = 'off' + - attributes['ng-ref'] = '$input2' +form-field__input({ name: name + `+"Text"`, model, disabled, required, placeholder })(attributes=attributes) password-visibility-toggle-button .form-field__errors( - ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}` + ng-messages=`$input1.$error || $input2.$error` + ng-show=` + ($input1.$dirty || $input1.$touched || $input1.$submitted) && $input1.$invalid || + ($input2.$dirty || $input2.$touched || $input2.$submitted) && $input2.$invalid + ` ) if block block diff --git a/modules/web-console/frontend/app/primitives/form-field/phone.pug b/modules/web-console/frontend/app/primitives/form-field/phone.pug index 8b503018d3bda..9b60a40457b0b 100644 --- a/modules/web-console/frontend/app/primitives/form-field/phone.pug +++ b/modules/web-console/frontend/app/primitives/form-field/phone.pug @@ -17,7 +17,7 @@ mixin form-field__phone({ label, model, name, disabled, required, optional, placeholder, tip }) -var errLbl = label.substring(0, label.length - 1) - .form-field + .form-field.ignite-form-field +form-field__label({ label, name, required, optional }) +form-field__tooltip({ title: tip, options: tipOpts }) @@ -26,7 +26,8 @@ mixin form-field__phone({ label, model, name, disabled, required, optional, plac +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes) .form-field__errors( - ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}` + ng-messages=`$input.$error` + ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) if block block diff --git a/modules/web-console/frontend/app/utils/dialogState.ts b/modules/web-console/frontend/app/utils/dialogState.ts new file mode 100644 index 0000000000000..d83c911035eb9 --- /dev/null +++ b/modules/web-console/frontend/app/utils/dialogState.ts @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {UIRouter, StateDeclaration, Transition} from '@uirouter/angularjs'; + +export function dialogState(component: string): Partial { + let dialog: mgcrea.ngStrap.modal.IModal | undefined; + let hide: (() => void) | undefined; + + onEnter.$inject = ['$transition$']; + + function onEnter(transition: Transition) { + const modal = transition.injector().get('$modal'); + const router = transition.injector().get('$uiRouter'); + + dialog = modal({ + template: ` + <${component} + class='modal center modal--ignite theme--ignite' + tabindex='-1' + role='dialog' + on-hide=$hide() + > + `, + onHide(modal) { + modal.destroy(); + } + }); + + hide = dialog.hide; + + dialog.hide = () => router.stateService.go('.^'); + } + + return { + onEnter, + onExit() { + if (hide) hide(); + dialog = hide = void 0; + } + }; +} From c1bd978a804386dd7850b7ccc584399d6b9eb851 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 29 Oct 2018 23:01:41 +0700 Subject: [PATCH 134/403] IGNITE-9941 Fixed typo. (cherry picked from commit e0cbaec4d260c6e43fec09ed09119b9db2070d65) --- modules/web-console/backend/config/settings.json.sample | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/web-console/backend/config/settings.json.sample b/modules/web-console/backend/config/settings.json.sample index 4070163c26b4b..aa20ab46f5c92 100644 --- a/modules/web-console/backend/config/settings.json.sample +++ b/modules/web-console/backend/config/settings.json.sample @@ -8,7 +8,8 @@ "keyPassphrase": "password", "disable": { "signup": false - } }, + } + }, "mongodb": { "url": "mongodb://localhost/console" }, From 1af25210e82af98648024c41d10f108fe69ce60b Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Tue, 30 Oct 2018 22:22:38 +0700 Subject: [PATCH 135/403] IGNITE-9941 Fixed E2E tests. (cherry picked from commit 99bb28f2158ff74b264db7e7ae44ecedf05f8a76) --- .../e2e/testcafe/page-models/pageSignup.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/web-console/e2e/testcafe/page-models/pageSignup.js b/modules/web-console/e2e/testcafe/page-models/pageSignup.js index c380abed41907..83f2932b2c362 100644 --- a/modules/web-console/e2e/testcafe/page-models/pageSignup.js +++ b/modules/web-console/e2e/testcafe/page-models/pageSignup.js @@ -19,13 +19,13 @@ import {Selector, t} from 'testcafe'; import {CustomFormField} from '../components/FormField'; export const pageSignup = { - email: new CustomFormField({model: '$ctrl.data.email'}), - password: new CustomFormField({model: '$ctrl.data.password'}), - passwordConfirm: new CustomFormField({model: 'confirm'}), - firstName: new CustomFormField({model: '$ctrl.data.firstName'}), - lastName: new CustomFormField({model: '$ctrl.data.lastName'}), - company: new CustomFormField({model: '$ctrl.data.company'}), - country: new CustomFormField({model: '$ctrl.data.country'}), + email: new CustomFormField({id: 'emailInput'}), + password: new CustomFormField({id: 'passwordInput'}), + passwordConfirm: new CustomFormField({id: 'confirmInput'}), + firstName: new CustomFormField({id: 'firstNameInput'}), + lastName: new CustomFormField({id: 'lastNameInput'}), + company: new CustomFormField({id: 'companyInput'}), + country: new CustomFormField({id: 'countryInput'}), signupButton: Selector('button').withText('Sign Up'), async fillSignupForm({ email, From 24574de55f322829dbd75a5376a50fe95eb305bd Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Fri, 2 Nov 2018 15:17:50 +0700 Subject: [PATCH 136/403] IGNITE-10119 Web Console: Remove "beta" badge. (cherry picked from commit de5e577c4969e86474b1e04b14d19d56a3109d1e) --- .../frontend/public/stylesheets/style.scss | 58 ------------------- modules/web-console/frontend/views/index.pug | 4 -- 2 files changed, 62 deletions(-) diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss index f9c9c4ad6a543..ce14e45d6ec0a 100644 --- a/modules/web-console/frontend/public/stylesheets/style.scss +++ b/modules/web-console/frontend/public/stylesheets/style.scss @@ -1869,64 +1869,6 @@ treecontrol.tree-classic { border-bottom-width: 0; } -.ribbon-wrapper { - width: 150px; - height: 150px; - position: absolute; - overflow: hidden; - top: 0; - z-index: 1001; - pointer-events: none; -} - -.ribbon-wrapper.right { - right: 0; -} - -.ribbon { - position: absolute; - top: 42px; - width: 186px; - padding: 1px 0; - color: $btn-primary-color; - background: $btn-primary-border; - - right: -42px; - -moz-transform: rotate(45deg); - -webkit-transform: rotate(45deg); - -o-transform: rotate(45deg); - -ms-transform: rotate(45deg); - transform: rotate(45deg); - - > label { - padding: 1px 0; - height: 17px; - line-height: 12px; - text-align: center; - text-decoration: none; - font-family: Roboto, sans-serif; - font-size: 16px; - display: block; - font-weight: 400; - - border: 1px solid rgba(255,255,255,0.3); - - -moz-text-shadow: 0 0 10px rgba(0,0,0,0.31); - -webkit-text-shadow: 0 0 10px rgba(0,0,0,0.31); - text-shadow: 0 0 10px rgba(0,0,0,0.31); - } -} - -.ribbon-wrapper.left { - overflow: visible; - - .ribbon { - transform: rotate(-45deg); - left: -75px; - top: 10px; - } -} - html, body { width: 100%; min-height: 100vh; diff --git a/modules/web-console/frontend/views/index.pug b/modules/web-console/frontend/views/index.pug index 647f1a225ae23..a0c68103a120b 100644 --- a/modules/web-console/frontend/views/index.pug +++ b/modules/web-console/frontend/views/index.pug @@ -41,8 +41,4 @@ html .splash-wellcome Loading... - .ribbon-wrapper.left(ng-if='!IgniteDemoMode') - .ribbon - label Beta - .wrapper(ui-view='') From 6270db2639e6e220b9cbad8e9bca906d8e811e08 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Tue, 6 Nov 2018 16:41:08 +0700 Subject: [PATCH 137/403] IGNITE-10101 Web Console: Refactored logic of collecting cache names for "Queries" screen. (cherry picked from commit 86b2351048e182ed71de67336e5e51309f56ff25) --- .../backend/app/browsersHandler.js | 5 + .../components/queries-notebook/controller.js | 138 +++++++----------- .../queries-notebook/template.tpl.pug | 4 +- .../components/page-queries/notebook.data.js | 3 - .../app/modules/agent/AgentManager.service.js | 52 ++++++- 5 files changed, 110 insertions(+), 92 deletions(-) diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js index a18b4673d851b..a7156c5281315 100644 --- a/modules/web-console/backend/app/browsersHandler.js +++ b/modules/web-console/backend/app/browsersHandler.js @@ -244,6 +244,11 @@ module.exports = { this.registerVisorTask('toggleClusterState', internalVisor('misc.VisorChangeGridActiveStateTask'), internalVisor('misc.VisorChangeGridActiveStateTaskArg')); + this.registerVisorTask('cacheNamesCollectorTask', internalVisor('cache.VisorCacheNamesCollectorTask'), 'java.lang.Void'); + + this.registerVisorTask('cacheNodesTask', internalVisor('cache.VisorCacheNodesTask'), 'java.lang.String'); + this.registerVisorTask('cacheNodesTaskX2', internalVisor('cache.VisorCacheNodesTask'), internalVisor('cache.VisorCacheNodesTaskArg')); + // Return command result from grid to browser. sock.on('node:visor', (arg, cb) => { const {clusterId, params, credentials} = arg || {}; diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js index 3ee08b31a9e56..2cb328ca50f03 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js @@ -14,6 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import _ from 'lodash'; import {nonEmpty, nonNil} from 'app/utils/lodashMixins'; import id8 from 'app/utils/id8'; import 'rxjs/add/operator/mergeMap'; @@ -891,37 +893,13 @@ export class NotebookCtrl { /** * Update caches list. */ - const _refreshFn = () => { - return agentMgr.topology(true) - .then((nodes) => { - $scope.caches = _.sortBy(_.reduce(nodes, (cachesAcc, node) => { - _.forEach(node.caches, (cache) => { - let item = _.find(cachesAcc, {name: cache.name}); - - if (_.isNil(item)) { - cache.label = maskCacheName(cache.name, true); - cache.value = cache.name; - - cache.nodes = []; - - cachesAcc.push(item = cache); - } - - item.nodes.push({ - nid: node.nodeId.toUpperCase(), - ip: _.head(node.attributes['org.apache.ignite.ips'].split(', ')), - version: node.attributes['org.apache.ignite.build.ver'], - gridName: node.attributes['org.apache.ignite.ignite.name'], - os: `${node.attributes['os.name']} ${node.attributes['os.arch']} ${node.attributes['os.version']}`, - client: node.attributes['org.apache.ignite.cache.client'] - }); - }); - - return cachesAcc; - }, []), (cache) => cache.label.toLowerCase()); - - // Reset to first cache in case of stopped selected. - const cacheNames = _.map($scope.caches, (cache) => cache.value); + const _refreshCaches = () => { + return agentMgr.publicCacheNames() + .then((cacheNames) => { + $scope.caches = _.sortBy(_.map(cacheNames, (name) => ({ + label: maskCacheName(name, true), + value: name + })), (cache) => cache.label.toLowerCase()); _.forEach($scope.notebook.paragraphs, (paragraph) => { if (!_.includes(cacheNames, paragraph.cacheName)) @@ -936,6 +914,8 @@ export class NotebookCtrl { _.forEach($scope.notebook.paragraphs, (paragraph) => $scope.execute(paragraph)); } + + $scope.$applyAsync(); }) .catch((err) => Messages.showError(err)); }; @@ -950,7 +930,7 @@ export class NotebookCtrl { }).take(1); const refreshCaches = (period) => { - return timer(0, period).exhaustMap(() => _refreshFn()).merge(finishLoading$); + return timer(0, period).exhaustMap(() => _refreshCaches()).merge(finishLoading$); }; this.refresh$ = awaitClusters$ @@ -1390,29 +1370,48 @@ export class NotebookCtrl { /** * @param {String} name Cache name. - * @return {Array.} Nids + * @param {Array.} nids Cache name. + * @return {Promise>} */ - const cacheNodes = (name) => { - return _.find($scope.caches, {name}).nodes; + const cacheNodesModel = (name, nids) => { + return agentMgr.topology(true) + .then((nodes) => + _.reduce(nodes, (acc, node) => { + if (_.includes(nids, node.nodeId)) { + acc.push({ + nid: node.nodeId.toUpperCase(), + ip: _.head(node.attributes['org.apache.ignite.ips'].split(', ')), + version: node.attributes['org.apache.ignite.build.ver'], + gridName: node.attributes['org.apache.ignite.ignite.name'], + os: `${node.attributes['os.name']} ${node.attributes['os.arch']} ${node.attributes['os.version']}`, + client: node.attributes['org.apache.ignite.cache.client'] + }); + } + + return acc; + }, []) + ); }; /** - * @param {String} name Cache name. - * @param {Boolean} local Local query. - * @return {String} Nid + * @param {string} name Cache name. + * @param {boolean} local Local query. + * @return {Promise} Nid */ const _chooseNode = (name, local) => { if (_.isEmpty(name)) return Promise.resolve(null); - const nodes = _.filter(cacheNodes(name), (node) => !node.client); - - if (local) { - return Nodes.selectNode(nodes, name) - .then((selectedNids) => _.head(selectedNids)); - } + return agentMgr.cacheNodes(name) + .then((nids) => { + if (local) { + return cacheNodesModel(name, nids) + .then((nodes) => Nodes.selectNode(nodes, name).catch(() => {})) + .then((selectedNids) => _.head(selectedNids)); + } - return Promise.resolve(nodes[_.random(0, nodes.length - 1)].nid); + return nids[_.random(0, nids.length - 1)]; + }); }; const _executeRefresh = (paragraph) => { @@ -1446,53 +1445,28 @@ export class NotebookCtrl { ${query} ) LIMIT ${limitSize}`; - $scope.nonCollocatedJoinsAvailable = (paragraph) => { - const cache = _.find($scope.caches, {name: paragraph.cacheName}); - - if (cache) - return !!_.find(cache.nodes, (node) => Version.since(node.version, NON_COLLOCATED_JOINS_SINCE)); - - return false; + $scope.nonCollocatedJoinsAvailable = () => { + return Version.since(this.agentMgr.clusterVersion, NON_COLLOCATED_JOINS_SINCE); }; - $scope.collocatedJoinsAvailable = (paragraph) => { - const cache = _.find($scope.caches, {name: paragraph.cacheName}); - - if (cache) - return !!_.find(cache.nodes, (node) => Version.since(node.version, ...COLLOCATED_QUERY_SINCE)); - - return false; + $scope.collocatedJoinsAvailable = () => { + return Version.since(this.agentMgr.clusterVersion, ...COLLOCATED_QUERY_SINCE); }; - $scope.enforceJoinOrderAvailable = (paragraph) => { - const cache = _.find($scope.caches, {name: paragraph.cacheName}); - - if (cache) - return !!_.find(cache.nodes, (node) => Version.since(node.version, ...ENFORCE_JOIN_SINCE)); - - return false; + $scope.enforceJoinOrderAvailable = () => { + return Version.since(this.agentMgr.clusterVersion, ...ENFORCE_JOIN_SINCE); }; - $scope.lazyQueryAvailable = (paragraph) => { - const cache = _.find($scope.caches, {name: paragraph.cacheName}); - - if (cache) - return !!_.find(cache.nodes, (node) => Version.since(node.version, ...LAZY_QUERY_SINCE)); - - return false; + $scope.lazyQueryAvailable = () => { + return Version.since(this.agentMgr.clusterVersion, ...LAZY_QUERY_SINCE); }; - $scope.ddlAvailable = (paragraph) => { - const cache = _.find($scope.caches, {name: paragraph.cacheName}); - - if (cache) - return !!_.find(cache.nodes, (node) => Version.since(node.version, ...DDL_SINCE)); - - return false; + $scope.ddlAvailable = () => { + return Version.since(this.agentMgr.clusterVersion, ...DDL_SINCE); }; $scope.cacheNameForSql = (paragraph) => { - return $scope.ddlAvailable(paragraph) && !paragraph.useAsDefaultSchema ? null : paragraph.cacheName; + return $scope.ddlAvailable() && !paragraph.useAsDefaultSchema ? null : paragraph.cacheName; }; $scope.execute = (paragraph, local = false) => { @@ -1893,7 +1867,7 @@ export class NotebookCtrl { agentMgr.metadata() .then((metadata) => { $scope.metadata = _.sortBy(_.filter(metadata, (meta) => { - const cache = _.find($scope.caches, { name: meta.cacheName }); + const cache = _.find($scope.caches, { value: meta.cacheName }); if (cache) { meta.name = (cache.sqlSchema || '"' + meta.cacheName + '"') + '.' + meta.typeName; diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug index 781ce5190268f..af673a822d9fb 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug @@ -394,12 +394,12 @@ mixin paragraph-query input.form-control(type='text' st-search='label' placeholder='Filter caches...') .queries-notebook-displayed-caches - div(ng-repeat='cache in displayedCaches track by cache.name') + div(ng-repeat='cache in displayedCaches track by cache.value') +form-field__radio({ label: '{{ cache.label }}', model: 'paragraph.cacheName', name: '"cache_" + [paragraph.id, $index].join("_")', - value: 'cache.name' + value: 'cache.value' }) .settings-row diff --git a/modules/web-console/frontend/app/components/page-queries/notebook.data.js b/modules/web-console/frontend/app/components/page-queries/notebook.data.js index 6c784bfc3ea11..d9035c2445621 100644 --- a/modules/web-console/frontend/app/components/page-queries/notebook.data.js +++ b/modules/web-console/frontend/app/components/page-queries/notebook.data.js @@ -21,7 +21,6 @@ const DEMO_NOTEBOOK = { paragraphs: [ { name: 'Query with refresh rate', - cacheName: 'CarCache', qryType: 'query', pageSize: 100, limit: 0, @@ -39,7 +38,6 @@ const DEMO_NOTEBOOK = { }, { name: 'Simple query', - cacheName: 'CarCache', qryType: 'query', pageSize: 100, limit: 0, @@ -54,7 +52,6 @@ const DEMO_NOTEBOOK = { }, { name: 'Query with aggregates', - cacheName: 'ParkingCache', qryType: 'query', pageSize: 100, limit: 0, diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index 7226cff988ecc..c59d6a130434b 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -43,6 +43,15 @@ const State = { const IGNITE_2_0 = '2.0.0'; const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1']; const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], ['2.5.1-p13', '2.6.0'], '2.7.0']; +const COLLECT_BY_CACHE_GROUPS_SINCE = '2.7.0'; + +/** Reserved cache names */ +const RESERVED_CACHE_NAMES = [ + 'ignite-hadoop-mr-sys-cache', + 'ignite-sys-cache', + 'MetaStorage', + 'TxLog' +]; /** Error codes from o.a.i.internal.processors.restGridRestResponse.java */ const SuccessStatus = { @@ -125,7 +134,7 @@ export default class AgentManager { ClusterLoginSrv; /** @type {String} */ - clusterVersion = '2.4.0'; + clusterVersion; connectionSbj = new BehaviorSubject(new ConnectionState(AgentManager.restoreActiveCluster())); @@ -171,6 +180,8 @@ export default class AgentManager { this.Version = Version; this.ClusterLoginSrv = ClusterLoginSrv; + this.clusterVersion = this.Version.webConsole; + let prevCluster; this.currentCluster$ = this.connectionSbj @@ -537,12 +548,43 @@ export default class AgentManager { } /** - * @param {Boolean} [attr] - * @param {Boolean} [mtr] + * @param {boolean} [attr] Collect node attributes. + * @param {boolean} [mtr] Collect node metrics. + * @param {boolean} [caches] Collect node caches descriptors. * @returns {Promise} */ - topology(attr = false, mtr = false) { - return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches: false}); + topology(attr = false, mtr = false, caches = false) { + return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches}); + } + + collectCacheNames(nid) { + if (this.available(COLLECT_BY_CACHE_GROUPS_SINCE)) + return this.visorTask('cacheNamesCollectorTask', nid); + + return Promise.resolve({cacheGroupsNotAvailable: true}); + } + + publicCacheNames() { + return this.collectCacheNames() + .then((data) => { + if (nonEmpty(data.caches)) + return _.difference(_.keys(data.caches), RESERVED_CACHE_NAMES); + + return this.topology(false, false, true) + .then((nodes) => { + return _.map(_.uniqBy(_.flatMap(nodes, 'caches'), 'name'), 'name'); + }); + }); + } + + /** + * @param {string} cacheName Cache name. + */ + cacheNodes(cacheName) { + if (this.available(IGNITE_2_0)) + return this.visorTask('cacheNodesTaskX2', null, cacheName); + + return this.visorTask('cacheNodesTask', null, cacheName); } /** From cae8076bf07e08819e720a50b654035b006a0cfa Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Thu, 15 Nov 2018 10:02:40 +0700 Subject: [PATCH 138/403] IGNITE-6600 Web Console: Refactored 404 & 403 pages to timed redirection component. (cherry picked from commit efd2c9c72f3a98ac7f7401bde091c0b645c8d133) --- modules/web-console/frontend/app/app.js | 4 +- .../components/timed-redirection/component.ts | 29 +++++++++ .../timed-redirection/controller.ts | 60 +++++++++++++++++++ .../app/components/timed-redirection/index.ts | 23 +++++++ .../components/timed-redirection/style.scss | 47 +++++++++++++++ .../timed-redirection/template.pug} | 10 ++-- .../app/modules/states/errors.state.js | 18 ++++-- .../web-console/frontend/views/404.tpl.pug | 24 -------- 8 files changed, 178 insertions(+), 37 deletions(-) create mode 100644 modules/web-console/frontend/app/components/timed-redirection/component.ts create mode 100644 modules/web-console/frontend/app/components/timed-redirection/controller.ts create mode 100644 modules/web-console/frontend/app/components/timed-redirection/index.ts create mode 100644 modules/web-console/frontend/app/components/timed-redirection/style.scss rename modules/web-console/frontend/{views/403.tpl.pug => app/components/timed-redirection/template.pug} (75%) delete mode 100644 modules/web-console/frontend/views/404.tpl.pug diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index 47374aa31fbca..2dd2733fc5689 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -150,6 +150,7 @@ import formField from './components/form-field'; import igniteChart from './components/ignite-chart'; import igniteChartSelector from './components/ignite-chart-series-selector'; import statusOutput from './components/status-output'; +import timedRedirection from './components/timed-redirection'; import pageProfile from './components/page-profile'; import pagePasswordChanged from './components/page-password-changed'; @@ -264,7 +265,8 @@ export default angular.module('ignite-console', [ statusOutput.name, progressLine.name, formField.name, - formSignup.name + formSignup.name, + timedRedirection.name ]) .service('$exceptionHandler', $exceptionHandler) // Directives. diff --git a/modules/web-console/frontend/app/components/timed-redirection/component.ts b/modules/web-console/frontend/app/components/timed-redirection/component.ts new file mode 100644 index 0000000000000..5cab0c9dd6300 --- /dev/null +++ b/modules/web-console/frontend/app/components/timed-redirection/component.ts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import './style.scss'; +import {TimedRedirectionCtrl} from './controller'; + +export const component: ng.IComponentOptions = { + template, + controller: TimedRedirectionCtrl, + bindings: { + headerText: '<', + subHeaderText: '<' + } +}; diff --git a/modules/web-console/frontend/app/components/timed-redirection/controller.ts b/modules/web-console/frontend/app/components/timed-redirection/controller.ts new file mode 100644 index 0000000000000..034ff3c20f011 --- /dev/null +++ b/modules/web-console/frontend/app/components/timed-redirection/controller.ts @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {StateOrName, StateService} from '@uirouter/angularjs'; +import {RawParams} from '@uirouter/core/lib/params/interface'; + +interface State { + name: StateOrName, + params: RawParams +} + +export class TimedRedirectionCtrl implements ng.IComponentController, ng.IOnInit, ng.IOnDestroy { + static $inject = ['$state', '$interval']; + + lastSuccessState = JSON.parse(localStorage.getItem('lastStateChangeSuccess')); + + stateToGo: State = this.lastSuccessState || {name: 'default-state', params: {}}; + + secondsLeft: number = 10; + + countDown: ng.IPromise; + + constructor(private $state: StateService, private $interval: ng.IIntervalService) {} + + $onInit() { + this.startCountDown(); + } + + $onDestroy() { + this.$interval.cancel(this.countDown); + } + + go(): void { + this.$state.go(this.stateToGo.name, this.stateToGo.params); + } + + startCountDown(): void { + this.countDown = this.$interval(() => { + this.secondsLeft--; + + if (this.secondsLeft === 0) + this.go(); + + }, 1000, this.secondsLeft); + } +} diff --git a/modules/web-console/frontend/app/components/timed-redirection/index.ts b/modules/web-console/frontend/app/components/timed-redirection/index.ts new file mode 100644 index 0000000000000..c4951f6c8edde --- /dev/null +++ b/modules/web-console/frontend/app/components/timed-redirection/index.ts @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as angular from 'angular'; + +import {component} from './component'; + +export default angular.module('ignite-console.timed-redirection', []) + .component('timedRedirection', component); diff --git a/modules/web-console/frontend/app/components/timed-redirection/style.scss b/modules/web-console/frontend/app/components/timed-redirection/style.scss new file mode 100644 index 0000000000000..5b0ecfd417f6c --- /dev/null +++ b/modules/web-console/frontend/app/components/timed-redirection/style.scss @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "./../../../public/stylesheets/variables.scss"; + +timed-redirection { + justify-content: center; + align-items: center; + display: flex; + flex-direction: column; + flex: 1; + + .timed-redirection--wrapper { + text-align: center; + } + + h1, .sub-header { + color: $ignite-brand-primary; + } + + h1 { + font-size: 64px; + } + + .sub-header { + font-size: 30px; + } + + .redirection-text { + margin-top: 30px; + font-size: 16px; + } +} diff --git a/modules/web-console/frontend/views/403.tpl.pug b/modules/web-console/frontend/app/components/timed-redirection/template.pug similarity index 75% rename from modules/web-console/frontend/views/403.tpl.pug rename to modules/web-console/frontend/app/components/timed-redirection/template.pug index 491ff6b622651..77f8298158b74 100644 --- a/modules/web-console/frontend/views/403.tpl.pug +++ b/modules/web-console/frontend/app/components/timed-redirection/template.pug @@ -14,11 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. -web-console-header +.timed-redirection--wrapper + h1 {{::$ctrl.headerText}} -.container--responsive.body-container - .error-page - h1.error-page__title 403 - h2.error-page__description You are not authorized + .sub-header {{::$ctrl.subHeaderText}} -web-console-footer \ No newline at end of file + .redirection-text You’ll be redirected back automatically in {{$ctrl.secondsLeft}} seconds, or #[a(ng-click='$ctrl.go()') click here] to redirect now. diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js index 19a1947b60334..0311b41695a82 100644 --- a/modules/web-console/frontend/app/modules/states/errors.state.js +++ b/modules/web-console/frontend/app/modules/states/errors.state.js @@ -16,8 +16,6 @@ */ import angular from 'angular'; -import templateNotFoundPage from 'views/404.tpl.pug'; -import templateNotAuthorizedPage from 'views/403.tpl.pug'; angular .module('ignite-console.states.errors', [ @@ -26,17 +24,25 @@ angular .config(['$stateProvider', /** @param {import('@uirouter/angularjs').StateProvider} $stateProvider */ function($stateProvider) { // set up the states $stateProvider - .state('404', { + .state('base.404', { url: '/404', - templateUrl: templateNotFoundPage, + component: 'timedRedirection', + resolve: { + headerText: () => '404', + subHeaderText: () => 'Page not found' + }, tfMetaTags: { title: 'Page not found' }, unsaved: true }) - .state('403', { + .state('base.403', { url: '/403', - templateUrl: templateNotAuthorizedPage, + component: 'timedRedirection', + resolve: { + headerText: () => '403', + subHeaderText: () => 'You are not authorized' + }, tfMetaTags: { title: 'Not authorized' }, diff --git a/modules/web-console/frontend/views/404.tpl.pug b/modules/web-console/frontend/views/404.tpl.pug deleted file mode 100644 index d9a0795dc73ad..0000000000000 --- a/modules/web-console/frontend/views/404.tpl.pug +++ /dev/null @@ -1,24 +0,0 @@ -//- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -web-console-header - -.container--responsive.body-container - .error-page - h1.error-page__title 404 - h2.error-page__description Page not found - -web-console-footer \ No newline at end of file From 89cab64af7e2d7dfba080e787dbd04ddf2f395db Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Thu, 15 Nov 2018 10:38:25 +0700 Subject: [PATCH 139/403] IGNITE-9511 Web Console: Fixed issues with modals that contains Ace editor. (cherry picked from commit 5a240e705cc16b508321ca2c5a06c143e8c0b5b2) --- .../frontend/app/primitives/modal/index.scss | 26 +++++++++++++++++-- modules/web-console/frontend/package.json | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/modules/web-console/frontend/app/primitives/modal/index.scss b/modules/web-console/frontend/app/primitives/modal/index.scss index 2be2ecdc2c276..eb621e531122a 100644 --- a/modules/web-console/frontend/app/primitives/modal/index.scss +++ b/modules/web-console/frontend/app/primitives/modal/index.scss @@ -54,7 +54,6 @@ .modal .modal-dialog { width: 650px; - transform: translateZ(1px); } .modal .modal-dialog--adjust-height { @@ -247,4 +246,27 @@ .modal--wide .modal-dialog { width: 900px; -} \ No newline at end of file +} + +.modal-with-ace { + .modal-body { + padding-left: 5px !important; + } + + .modal-dialog { + transform: none !important; + + .ace_warning:before, .ace_error:before { + position: absolute !important; + left: -7px !important; + } + } + + .ace_gutter, .ace_layer { + overflow: visible !important; + } + + .ace_gutter { + padding-left: 5px !important; + } +} diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json index 2260f75a4e5d3..9059a5f3e2fd8 100644 --- a/modules/web-console/frontend/package.json +++ b/modules/web-console/frontend/package.json @@ -52,7 +52,7 @@ "angular-ui-grid": "4.6.1", "angular-ui-validate": "1.2.3", "angular1-async-filter": "1.1.0", - "brace": "0.10.0", + "brace": "0.11.1", "browser-update": "3.1.13", "bson-objectid": "1.1.5", "chart.js": "2.7.2", From 5610f45f98c303a589e31a7c2ffe9c723fb0d827 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Thu, 15 Nov 2018 11:06:17 +0700 Subject: [PATCH 140/403] IGNITE-7861 Web Console: Fixed "Activities" dialog UI and saving activities on state changing. (cherry picked from commit d196ce35590e7452015b4beea4697e650264e844) --- .../activities-user-dialog.controller.js | 6 ++--- .../activities-user-dialog.tpl.pug | 2 +- .../app/components/page-admin/index.js | 16 +++++++++++- .../app/components/page-configure/index.js | 11 ++++---- .../components/queries-notebook/controller.js | 10 ++++---- .../app/components/page-queries/index.js | 17 +++++++++++-- modules/web-console/frontend/app/data/i18n.js | 25 ++++++++++++------- .../frontend/app/modules/user/user.module.js | 8 ------ .../frontend/views/includes/header-left.pug | 2 +- 9 files changed, 62 insertions(+), 35 deletions(-) diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js index fb9d15a46b666..bd3bb0796669d 100644 --- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js +++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js @@ -25,9 +25,9 @@ export default class ActivitiesCtrl { $ctrl.data = _.map(user.activitiesDetail, (amount, action) => ({action, amount})); $ctrl.columnDefs = [ - { displayName: 'Description', field: 'action', enableFiltering: false, cellFilter: 'translate'}, - { displayName: 'Action', field: 'action', enableFiltering: false}, - { displayName: 'Visited', field: 'amount', enableFiltering: false} + { displayName: 'Description', field: 'action', enableFiltering: false, cellFilter: 'translate', minWidth: 120, width: '43%'}, + { displayName: 'Action', field: 'action', enableFiltering: false, minWidth: 120, width: '43%'}, + { displayName: 'Visited', field: 'amount', enableFiltering: false, minWidth: 80} ]; } } diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug index 7e9d3f5142364..511a37bfd9090 100644 --- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug +++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. -.modal.modal--ignite.theme--ignite(tabindex='-1' role='dialog') +.modal.modal--ignite.modal--wide.theme--ignite(tabindex='-1' role='dialog') .modal-dialog .modal-content .modal-header diff --git a/modules/web-console/frontend/app/components/page-admin/index.js b/modules/web-console/frontend/app/components/page-admin/index.js index 8b26f2c410290..69816f6823fef 100644 --- a/modules/web-console/frontend/app/components/page-admin/index.js +++ b/modules/web-console/frontend/app/components/page-admin/index.js @@ -19,10 +19,24 @@ import './style.scss'; import controller from './controller'; import templateUrl from './template.tpl.pug'; +import {default as ActivitiesData} from 'app/core/activities/Activities.data'; + +/** + * @param {uirouter.UIRouter} $uiRouter + * @param {ActivitiesData} ActivitiesData + */ +function registerActivitiesHook($uiRouter, ActivitiesData) { + $uiRouter.transitionService.onSuccess({to: 'base.settings.**'}, (transition) => { + ActivitiesData.post({group: 'settings', action: transition.targetState().name()}); + }); +} + +registerActivitiesHook.$inject = ['$uiRouter', 'IgniteActivitiesData']; export default angular .module('ignite-console.page-admin', []) .component('pageAdmin', { controller, templateUrl - }); + }) + .run(registerActivitiesHook); diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js index c1d51ee094be7..531b46cf679ac 100644 --- a/modules/web-console/frontend/app/components/page-configure/index.js +++ b/modules/web-console/frontend/app/components/page-configure/index.js @@ -19,7 +19,6 @@ import angular from 'angular'; import 'angular1-async-filter'; import {UIRouterRx} from '@uirouter/rx'; -import {Visualizer} from '@uirouter/visualizer'; import uiValidate from 'angular-ui-validate'; import component from './component'; @@ -54,6 +53,7 @@ import 'rxjs/add/operator/withLatestFrom'; import 'rxjs/add/operator/skip'; import {Observable} from 'rxjs/Observable'; + Observable.prototype.debug = function(l) { return this.do((v) => console.log(l, v), (e) => console.error(l, e), () => console.log(l, 'completed')); }; @@ -83,15 +83,16 @@ import {reducer as reduxDevtoolsReducer, devTools} from './reduxDevtoolsIntegrat import {registerStates} from './states'; /** - * @param {ActivitiesData} ActivitiesData * @param {uirouter.UIRouter} $uiRouter + * @param {ActivitiesData} ActivitiesData */ -function registerActivitiesHook(ActivitiesData, $uiRouter) { +function registerActivitiesHook($uiRouter, ActivitiesData) { $uiRouter.transitionService.onSuccess({to: 'base.configuration.**'}, (transition) => { ActivitiesData.post({group: 'configuration', action: transition.targetState().name()}); }); } -registerActivitiesHook.$inject = ['IgniteActivitiesData', '$uiRouter']; + +registerActivitiesHook.$inject = ['$uiRouter', 'IgniteActivitiesData']; export default angular .module('ignite-console.page-configure', [ @@ -116,7 +117,7 @@ export default angular .run(registerActivitiesHook) .run(['ConfigEffects', 'ConfigureState', '$uiRouter', (ConfigEffects, ConfigureState, $uiRouter) => { $uiRouter.plugin(UIRouterRx); - // $uiRouter.plugin(Visualizer); + if (devTools) { devTools.subscribe((e) => { if (e.type === 'DISPATCH' && e.state) ConfigureState.actions$.next(e); diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js index 2cb328ca50f03..4b1b8611a67fe 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js @@ -1038,7 +1038,7 @@ export class NotebookCtrl { $scope.addQuery = function() { const sz = $scope.notebook.paragraphs.length; - ActivitiesData.post({ action: '/queries/add/query' }); + ActivitiesData.post({ group: 'sql', action: '/queries/add/query' }); const paragraph = _newParagraph({ name: 'Query' + (sz === 0 ? '' : sz), @@ -1067,7 +1067,7 @@ export class NotebookCtrl { $scope.addScan = function() { const sz = $scope.notebook.paragraphs.length; - ActivitiesData.post({ action: '/queries/add/scan' }); + ActivitiesData.post({ group: 'sql', action: '/queries/add/scan' }); const paragraph = _newParagraph({ name: 'Scan' + (sz === 0 ? '' : sz), @@ -1503,7 +1503,7 @@ export class NotebookCtrl { collocated }; - ActivitiesData.post({ action: '/queries/execute' }); + ActivitiesData.post({ group: 'sql', action: '/queries/execute' }); const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : query; @@ -1562,7 +1562,7 @@ export class NotebookCtrl { pageSize: paragraph.pageSize }; - ActivitiesData.post({ action: '/queries/explain' }); + ActivitiesData.post({ group: 'sql', action: '/queries/explain' }); return agentMgr.querySql(nid, args.cacheName, args.query, nonCollocatedJoins, enforceJoinOrder, false, false, args.pageSize, false, collocated); }) @@ -1606,7 +1606,7 @@ export class NotebookCtrl { localNid: local ? nid : null }; - ActivitiesData.post({ action: '/queries/scan' }); + ActivitiesData.post({ group: 'sql', action: '/queries/scan' }); return agentMgr.queryScan(nid, cacheName, filter, false, caseSensitive, false, local, pageSize); }) diff --git a/modules/web-console/frontend/app/components/page-queries/index.js b/modules/web-console/frontend/app/components/page-queries/index.js index 9434dc19556a5..ba9e5114dd0c8 100644 --- a/modules/web-console/frontend/app/components/page-queries/index.js +++ b/modules/web-console/frontend/app/components/page-queries/index.js @@ -22,9 +22,21 @@ import angular from 'angular'; import queriesNotebooksList from './components/queries-notebooks-list'; import queriesNotebook from './components/queries-notebook'; import pageQueriesCmp from './component'; - +import {default as ActivitiesData} from 'app/core/activities/Activities.data'; import Notebook from './notebook.service'; +/** + * @param {uirouter.UIRouter} $uiRouter + * @param {ActivitiesData} ActivitiesData + */ +function registerActivitiesHook($uiRouter, ActivitiesData) { + $uiRouter.transitionService.onSuccess({to: 'base.sql.**'}, (transition) => { + ActivitiesData.post({group: 'sql', action: transition.targetState().name()}); + }); +} + +registerActivitiesHook.$inject = ['$uiRouter', 'IgniteActivitiesData']; + export default angular.module('ignite-console.sql', [ 'ui.router', queriesNotebooksList.name, @@ -84,4 +96,5 @@ export default angular.module('ignite-console.sql', [ title: 'Query notebook' } }); - }]); + }]) + .run(registerActivitiesHook); diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js index e6c1294dfce70..a3c2480bdc724 100644 --- a/modules/web-console/frontend/app/data/i18n.js +++ b/modules/web-console/frontend/app/data/i18n.js @@ -18,20 +18,12 @@ export default { '/agent/start': 'Agent start', '/agent/download': 'Agent download', - 'base.configuration.overview': 'Cluster configurations', '/configuration/overview': 'Cluster configurations', - 'base.configuration.edit.basic': 'Basic cluster configuration edit', '/configuration/new': 'Cluster configuration create', '/configuration/new/basic': 'Basic cluster configuration create', '/configuration/new/advanced/cluster': 'Advanced cluster configuration create', - 'base.configuration.edit.advanced.cluster': 'Advanced cluster configuration edit', - 'base.configuration.edit.advanced.caches': 'Advanced cluster caches', - 'base.configuration.edit.advanced.caches.cache': 'Advanced cluster cache edit', - 'base.configuration.edit.advanced.models': 'Advanced cluster models', - 'base.configuration.edit.advanced.models.model': 'Advanced cluster model edit', - 'base.configuration.edit.advanced.igfs': 'Advanced cluster IGFSs', - 'base.configuration.edit.advanced.igfs.igfs': 'Advanced cluster IGFS edit', '/configuration/download': 'Download project', + 'configuration/import/model': 'Import cluster models', '/demo/resume': 'Demo resume', '/demo/reset': 'Demo reset', '/queries/execute': 'Query execute', @@ -45,6 +37,20 @@ export default { '/settings/admin': 'Admin panel', '/logout': 'Logout', + 'base.configuration.overview': 'Cluster configurations', + 'base.configuration.edit.basic': 'Basic cluster configuration edit', + 'base.configuration.edit.advanced.cluster': 'Advanced cluster configuration edit', + 'base.configuration.edit.advanced.caches': 'Advanced cluster caches', + 'base.configuration.edit.advanced.caches.cache': 'Advanced cluster cache edit', + 'base.configuration.edit.advanced.models': 'Advanced cluster models', + 'base.configuration.edit.advanced.models.model': 'Advanced cluster model edit', + 'base.configuration.edit.advanced.igfs': 'Advanced cluster IGFSs', + 'base.configuration.edit.advanced.igfs.igfs': 'Advanced cluster IGFS edit', + 'base.sql.tabs.notebooks-list': 'Query notebooks', + 'base.sql.notebook': 'Query notebook', + 'base.settings.profile': 'User profile', + 'base.settings.admin': 'Admin panel', + // app/components/page-signin/template.pug 'app.components.page-signin.m1': 'Sign In', 'app.components.page-signin.m2': 'Email:', @@ -54,6 +60,7 @@ export default { 'app.components.page-signin.m6': 'Forgot password?', 'app.components.page-signin.m7': 'Sign In', 'app.components.page-signin.m8': 'Don\'t have an account? #[a(ui-sref=\'signup\') Get started]', + // app/components/page-signin/run.js 'app.components.page-signin.m9': 'Sign In', diff --git a/modules/web-console/frontend/app/modules/user/user.module.js b/modules/web-console/frontend/app/modules/user/user.module.js index 3e3068fe948fc..e11a2d471f6a9 100644 --- a/modules/web-console/frontend/app/modules/user/user.module.js +++ b/modules/web-console/frontend/app/modules/user/user.module.js @@ -90,14 +90,6 @@ function run($root, $transitions, AclService, User, Activities) { return $state.target(trans.to().failState || '403'); }); }); - - $transitions.onFinish({}, (trans) => { - const $state = trans.router.stateService; - const {name, permission} = trans.to(); - - if (AclService.can(permission)) - Activities.post({action: $state.href(name, trans.params('to'))}); - }); } run.$inject = ['$rootScope', '$transitions', 'AclService', 'User', 'IgniteActivitiesData']; diff --git a/modules/web-console/frontend/views/includes/header-left.pug b/modules/web-console/frontend/views/includes/header-left.pug index bf1ee9b71bd0f..03e06167360c0 100644 --- a/modules/web-console/frontend/views/includes/header-left.pug +++ b/modules/web-console/frontend/views/includes/header-left.pug @@ -21,7 +21,7 @@ a.wch-nav-item( | Configure .wch-nav-item(ng-if='!$root.user.becomeUsed') - a(ui-sref='base.sql.tabs' ui-sref-active='active') + a(ui-sref='base.sql.tabs.notebooks-list' ui-sref-active='active') | Queries .wch-additional-nav-items(ignite-navbar) From fc87a1906e0dbe447a9e8509e98494cc69ab03c4 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Thu, 15 Nov 2018 14:12:36 +0700 Subject: [PATCH 141/403] IGNITE-336 Visor CMD: Added commands to enable/disable cache statistics in runtime. (cherry picked from commit bf2f05e323d7eb6fbc0a1c5b9111ed8c05be503d) --- .../internal/visor/cache/VisorCache.java | 20 ++- .../cache/VisorCacheToggleStatisticsTask.java | 72 +++++++++ .../VisorCacheToggleStatisticsTaskArg.java | 87 ++++++++++ .../commands/cache/VisorCacheCommand.scala | 76 +++++---- .../VisorCacheToggleStatisticsCommand.scala | 152 ++++++++++++++++++ .../app/modules/cluster/CacheMetrics.js | 3 + 6 files changed, 380 insertions(+), 30 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTask.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTaskArg.java create mode 100644 modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheToggleStatisticsCommand.scala diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java index 63eb13c3cecc6..495a9cb6dbe7e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java @@ -91,6 +91,9 @@ public class VisorCache extends VisorDataTransferObject { /** Cache system state. */ private boolean sys; + /** Checks whether statistics collection is enabled in this cache. */ + private boolean statisticsEnabled; + /** * Create data transfer object for given cache. */ @@ -119,7 +122,7 @@ public VisorCache(IgniteEx ignite, GridCacheAdapter ca, boolean collectMetrics) backupSize = ca.localSizeLong(PEEK_ONHEAP_BACKUP); nearSize = ca.nearSize(); size = primarySize + backupSize + nearSize; - + partitions = ca.affinity().partitions(); near = cctx.isNear(); @@ -127,6 +130,8 @@ public VisorCache(IgniteEx ignite, GridCacheAdapter ca, boolean collectMetrics) metrics = new VisorCacheMetrics(ignite, name); sys = ignite.context().cache().systemCache(name); + + statisticsEnabled = ca.clusterMetrics().isStatisticsEnabled(); } /** @@ -278,9 +283,16 @@ public long offHeapEntriesCount() { return metrics != null ? metrics.getOffHeapEntriesCount() : 0L; } + /** + * @return Checks whether statistics collection is enabled in this cache. + */ + public boolean isStatisticsEnabled() { + return statisticsEnabled; + } + /** {@inheritDoc} */ @Override public byte getProtocolVersion() { - return V2; + return V3; } /** {@inheritDoc} */ @@ -298,6 +310,7 @@ public long offHeapEntriesCount() { out.writeBoolean(near); out.writeObject(metrics); out.writeBoolean(sys); + out.writeBoolean(statisticsEnabled); } /** {@inheritDoc} */ @@ -316,6 +329,9 @@ public long offHeapEntriesCount() { metrics = (VisorCacheMetrics)in.readObject(); sys = protoVer > V1 ? in.readBoolean() : metrics != null && metrics.isSystem(); + + if (protoVer > V2) + statisticsEnabled = in.readBoolean(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTask.java new file mode 100644 index 0000000000000..aebed810d2cc0 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTask.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.cache; + +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; + +/** + * Switch statisticsEnabled flag for specified caches to specified state. + */ +@GridInternal +public class VisorCacheToggleStatisticsTask extends VisorOneNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorCachesToggleStatisticsJob job(VisorCacheToggleStatisticsTaskArg arg) { + return new VisorCachesToggleStatisticsJob(arg, debug); + } + + /** + * Job that switch statisticsEnabled flag for specified caches to specified state. + */ + private static class VisorCachesToggleStatisticsJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Job argument object. + * @param debug Debug flag. + */ + private VisorCachesToggleStatisticsJob(VisorCacheToggleStatisticsTaskArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Void run(VisorCacheToggleStatisticsTaskArg arg) { + try { + ignite.context().cache().enableStatistics(arg.getCacheNames(), arg.getState()); + } + catch (IgniteCheckedException e) { + throw U.convertException(e); + } + + return null; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorCachesToggleStatisticsJob.class, this); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTaskArg.java new file mode 100644 index 0000000000000..34359daeacf6c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheToggleStatisticsTaskArg.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.cache; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Set; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * Argument for {@link VisorCacheToggleStatisticsTask}. + */ +public class VisorCacheToggleStatisticsTaskArg extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** State to set to statisticsEnabled flag. */ + private boolean state; + + /** Cache names to toggle statisticsEnabled flag. */ + private Set cacheNames; + + /** + * Default constructor. + */ + public VisorCacheToggleStatisticsTaskArg() { + // No-op. + } + + /** + * @param state State to set to statisticsEnabled flag. + * @param cacheNames Collection of cache names to toggle statisticsEnabled flag. + */ + public VisorCacheToggleStatisticsTaskArg(boolean state, Set cacheNames) { + this.state = state; + this.cacheNames = cacheNames; + } + + /** + * @return State to set to statisticsEnabled flag. + */ + public boolean getState() { + return state; + } + + /** + * @return Cache names to toggle statisticsEnabled flag. + */ + public Set getCacheNames() { + return cacheNames; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + out.writeBoolean(state); + U.writeCollection(out, cacheNames); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + state = in.readBoolean(); + cacheNames = U.readSet(in); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorCacheToggleStatisticsTaskArg.class, this); + } +} diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala index 25f1212720d99..5853610448c17 100755 --- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala +++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala @@ -41,31 +41,33 @@ import scala.language.{implicitConversions, reflectiveCalls} * * ==Help== * {{{ - * +-------------------------------------------------------------------------------------------+ - * | cache | Prints statistics about caches from specified node on the entire grid. | - * | | Output sorting can be specified in arguments. | - * | | | - * | | Output abbreviations: | - * | | # Number of nodes. | - * | | H/h Number of cache hits. | - * | | M/m Number of cache misses. | - * | | R/r Number of cache reads. | - * | | W/w Number of cache writes. | - * +-------------------------------------------------------------------------------------------+ - * | cache -clear | Clears all entries from cache on all nodes. | - * +-------------------------------------------------------------------------------------------+ - * | cache -scan | List all entries in cache with specified name. | - * +-------------------------------------------------------------------------------------------+ - * | cache -stop | Stop cache with specified name. | - * +-------------------------------------------------------------------------------------------+ - * | cache -reset | Reset metrics for cache with specified name. | - * +-------------------------------------------------------------------------------------------+ - * | cache -rebalance | Re-balance partitions for cache with specified name. | - * +-------------------------------------------------------------------------------------------+ - * | cache -slp | Show list of lost partitions for specified cache. | - * +-------------------------------------------------------------------------------------------+ - * | cache -rlp | Reset lost partitions for specified cache. | - * +-------------------------------------------------------------------------------------------+ + * +--------------------------------------------------------------------------------------------+ + * | cache | Prints statistics about caches from specified node on the entire grid. | + * | | Output sorting can be specified in arguments. | + * | | | + * | | Output abbreviations: | + * | | # Number of nodes. | + * | | H/h Number of cache hits. | + * | | M/m Number of cache misses. | + * | | R/r Number of cache reads. | + * | | W/w Number of cache writes. | + * +--------------------------------------------------------------------------------------------+ + * | cache -clear | Clears all entries from cache on all nodes. | + * +--------------------------------------------------------------------------------------------+ + * | cache -scan | List all entries in cache with specified name. | + * +--------------------------------------------------------------------------------------------+ + * | cache -stop | Stop cache with specified name. | + * +--------------------------------------------------------------------------------------------+ + * | cache -reset | Reset metrics for cache with specified name. | + * +--------------------------------------------------------------------------------------------+ + * | cache -rebalance | Re-balance partitions for cache with specified name. | + * +--------------------------------------------------------------------------------------------+ + * | cache -slp | Show list of lost partitions for specified cache. | + * +--------------------------------------------------------------------------------------------+ + * | cache -rlp | Reset lost partitions for specified cache. | + * +--------------------------------------------------------------------------------------------+ + * | cache -statistics | Switch collection of statistics for specified cache. | + * +--------------------------------------------------------------------------------------------+ * * }}} * @@ -81,6 +83,7 @@ import scala.language.{implicitConversions, reflectiveCalls} * cache -rebalance -c= * cache -slp -c= * cache -rlp -c= + * cache -statistics= -c= * }}} * * ====Arguments==== @@ -133,6 +136,8 @@ import scala.language.{implicitConversions, reflectiveCalls} * Show list of lost partitions for specified cache. * -rlp * Reset lost partitions for specified cache. + * -statistics=<-on|-off> + * Switch collection of statistics for specified cache. * -p= * Number of object to fetch from cache at once. * Valid range from 1 to 100. @@ -177,6 +182,10 @@ import scala.language.{implicitConversions, reflectiveCalls} * Show list of lost partitions for cache with name 'cache'. * cache -rlp -c=cache * Reset lost partitions for cache with name 'cache'. + * cache -statistics=on -c=cache + * Enable statistics for cache with name 'cache'. + * cache -statistics=off -c=cache + * Disable statistics for cache with name 'cache'. * * }}} */ @@ -279,7 +288,8 @@ class VisorCacheCommand extends VisorConsoleCommand { // Get cache stats data from all nodes. val aggrData = cacheData(node, cacheName, showSystem) - if (hasArgFlagIn("clear", "scan", "stop", "reset", "rebalance", "slp", "rlp")) { + if (hasArgFlagIn("clear", "scan", "stop", "reset", "rebalance", "slp", "rlp") + || hasArgName("statistics", argLst)) { if (cacheName.isEmpty) askForCache("Select cache from:", node, showSystem && !hasArgFlagIn("clear", "stop", "reset", "rebalance"), aggrData) match { @@ -309,6 +319,8 @@ class VisorCacheCommand extends VisorConsoleCommand { VisorCacheLostPartitionsCommand().showLostPartitions(argLst, node) else if (hasArgFlag("rlp", argLst)) VisorCacheResetLostPartitionsCommand().resetLostPartitions(argLst, node) + else if (hasArgName("statistics", argLst)) + VisorCacheToggleStatisticsCommand().toggle(argLst, node) } else { if (hasArgFlag("scan", argLst)) @@ -321,6 +333,8 @@ class VisorCacheCommand extends VisorConsoleCommand { warn("Reset metrics of system cache is not allowed: " + name) else if (hasArgFlag("rebalance", argLst)) warn("Re-balance partitions of system cache is not allowed: " + name) + else if (hasArgName("statistics", argLst)) + warn("Toggle of statistics collection for system cache is not allowed: " + name) } case None => warn("Cache with specified name not found: " + name) @@ -755,7 +769,8 @@ object VisorCacheCommand { "cache -reset -c=", "cache -rebalance -c=", "cache -slp -c=", - "cache -rlp -c=" + "cache -rlp -c=", + "cache -statistics= -c=" ), args = Seq( "-id8=" -> Seq( @@ -784,6 +799,7 @@ object VisorCacheCommand { "-slp" -> "Show list of lost partitions for specified cache.", "-rlp" -> "Reset lost partitions for specified cache.", "-rebalance" -> "Re-balance partitions for cache with specified name.", + "-statistics=" -> "Change statistics collection state for cache with specified name.", "-s=hi|mi|rd|wr|cn" -> Seq( "Defines sorting type. Sorted by:", " hi Hits.", @@ -845,7 +861,11 @@ object VisorCacheCommand { "cache -reset -c=@c0" -> "Reset metrics for cache with name taken from 'c0' memory variable.", "cache -rebalance -c=cache" -> "Re-balance partitions for cache with name 'cache'.", "cache -slp -c=@c0" -> "Show list of lost partitions for cache with name taken from 'c0' memory variable.", - "cache -rlp -c=@c0" -> "Reset lost partitions for cache with name taken from 'c0' memory variable." + "cache -rlp -c=@c0" -> "Reset lost partitions for cache with name taken from 'c0' memory variable.", + "cache -statistics=on -c=@c0" -> + "Enable statistics collection for cache with name taken from 'c0' memory variable.", + "cache -statistics=off -c=@c0" -> + "Disable statistics collection for cache with name taken from 'c0' memory variable." ), emptyArgs = cmd.cache, withArgs = cmd.cache diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheToggleStatisticsCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheToggleStatisticsCommand.scala new file mode 100644 index 0000000000000..2156de03390bd --- /dev/null +++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheToggleStatisticsCommand.scala @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.visor.commands.cache + +import java.util.{HashSet => JavaSet} + +import org.apache.ignite.cluster.{ClusterGroupEmptyException, ClusterNode} +import org.apache.ignite.internal.visor.cache.{VisorCacheToggleStatisticsTask, VisorCacheToggleStatisticsTaskArg} +import org.apache.ignite.internal.visor.util.VisorTaskUtils._ +import org.apache.ignite.visor.visor._ + +import scala.language.reflectiveCalls + +/** + * ==Overview== + * Visor 'statistics' command implementation. + * + * ====Specification==== + * {{{ + * cache -statistics= -c= + * }}} + * + * ====Arguments==== + * {{{ + * + * Statistics collection state to set for cache. + * + * Name of the cache. + * }}} + * + * ====Examples==== + * {{{ + * cache -statistics=on -c=@c0 + * Enable collection of statistics for cache with name taken from 'c0' memory variable. + * cache -statistics=off -c=@c0 + * Disable collection of statistics for cache with name taken from 'c0' memory variable. + * }}} + */ +class VisorCacheToggleStatisticsCommand { + /** + * Prints error message and advise. + * + * @param errMsgs Error messages. + */ + private def scold(errMsgs: Any*) { + assert(errMsgs != null) + + warn(errMsgs: _*) + warn("Type 'help cache' to see how to use this command.") + } + + private def error(e: Exception) { + var cause: Throwable = e + + while (cause.getCause != null) + cause = cause.getCause + + scold(cause.getMessage) + } + + /** + * ===Command=== + * Toggle statistics collection for cache with specified name. + * + * ===Examples=== + * cache -statistics=on -c=cache + * Enable collection of statistics for cache with name 'cache'. + * cache -statistics=off -c=cache + * Disable collection of statistics for cache with name 'cache'. + * + * @param argLst Command arguments. + */ + def toggle(argLst: ArgList, node: Option[ClusterNode]) { + val cacheArg = argValue("c", argLst) + + val cacheName = cacheArg match { + case None => null // default cache. + + case Some(s) if s.startsWith("@") => + warn("Can't find cache variable with specified name: " + s, + "Type 'cache' to see available cache variables." + ) + + return + + case Some(name) => name + } + + val grp = try { + groupForDataNode(node, cacheName) + } + catch { + case _: ClusterGroupEmptyException => + scold(messageNodeNotFound(node, cacheName)) + + return + } + + try { + val cacheNames = new JavaSet[String]() + cacheNames.add(cacheName) + + val enable = argValue("statistics", argLst) match { + case Some(state) if "on".equalsIgnoreCase(state) => true + case Some(state) if "off".equalsIgnoreCase(state) => false + case _ => + warn("Goal state for collection of cache statistics is not specified.", + "Use \"on\" and \"off\" value of \"statistics\" argument to toggle collection of cache statistics.") + + return + } + + executeRandom(grp, classOf[VisorCacheToggleStatisticsTask], + new VisorCacheToggleStatisticsTaskArg(enable, cacheNames)) + + println("Visor successfully " + (if (enable) "enable" else "disable") + + " collection of statistics for cache: " + escapeName(cacheName)) + } + catch { + case _: ClusterGroupEmptyException => scold(messageNodeNotFound(node, cacheName)) + case e: Exception => error(e) + } + } +} + +/** + * Companion object that does initialization of the command. + */ +object VisorCacheToggleStatisticsCommand { + /** Singleton command. */ + private val cmd = new VisorCacheToggleStatisticsCommand + + /** + * Singleton. + */ + def apply() = cmd +} diff --git a/modules/web-console/frontend/app/modules/cluster/CacheMetrics.js b/modules/web-console/frontend/app/modules/cluster/CacheMetrics.js index 90d3a9bd8adab..609b181bf2d8e 100644 --- a/modules/web-console/frontend/app/modules/cluster/CacheMetrics.js +++ b/modules/web-console/frontend/app/modules/cluster/CacheMetrics.js @@ -55,5 +55,8 @@ export default class CacheMetrics { // Transaction metrics. this.commits = m.txCommits; this.rollbacks = m.txRollbacks; + + // Admin metrics. + this.statisticsEnabled = m.statisticsEnabled; } } From 4eb3a5be9aff40a6be68897611eba87e56570b9d Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Tue, 20 Nov 2018 08:57:13 +0700 Subject: [PATCH 142/403] IGNITE-10229 Web Console: Cleanup ClusterLoginService on disconnect from cluster. (cherry picked from commit 351d0807529a96aa524ae7d02b08f33587c501b6) --- .../app/modules/agent/AgentManager.service.js | 11 ++++++++++- .../modules/agent/components/cluster-login/service.js | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index c59d6a130434b..dad625d8076f7 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -30,6 +30,7 @@ import Worker from './decompress.worker'; import SimpleWorkerPool from '../../utils/SimpleWorkerPool'; import maskNull from 'app/core/utils/maskNull'; +import {CancellationError} from 'app/errors/CancellationError'; import {ClusterSecretsManager} from './types/ClusterSecretsManager'; import ClusterLoginService from './components/cluster-login/service'; @@ -379,11 +380,13 @@ export default class AgentManager { case State.AGENT_DISCONNECTED: this.agentModal.agentDisconnected(this.backText, this.backState); + this.ClusterLoginSrv.cancel(); break; case State.CLUSTER_DISCONNECTED: this.agentModal.clusterDisconnected(this.backText, this.backState); + this.ClusterLoginSrv.cancel(); break; @@ -544,7 +547,13 @@ export default class AgentManager { return {cluster, credentials: {}}; }) - .then(({cluster, credentials}) => this._executeOnActiveCluster(cluster, credentials, event, params)); + .then(({cluster, credentials}) => this._executeOnActiveCluster(cluster, credentials, event, params)) + .catch((err) => { + if (err instanceof CancellationError) + return; + + return err; + }); } /** diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js index 955d0a3307e6c..406152df2011d 100644 --- a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js +++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js @@ -77,4 +77,9 @@ export default class ClusterLoginService { modal.hide(); }); } + + cancel() { + if (this.deferred) + this.deferred.reject(new CancellationError()); + } } From 0b534c1284b4ef76c88161a226b29d062023a246 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Tue, 20 Nov 2018 11:30:03 +0700 Subject: [PATCH 143/403] IGNITE-9339 Form-field-size improvements. - Fixes #5440. Signed-off-by: Andrey Novikov (cherry picked from commit 7c632e68d4b48480b22cdc950c67944415b8bd5e) --- .../e2e/testcafe/components/FormField.js | 11 +++++------ .../components/form-field-size/controller.js | 6 ++++++ .../components/form-field-size/index.js | 3 ++- .../components/form-field-size/style.scss | 15 +++++++++++++++ .../components/form-field-size/template.pug | 1 + 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/modules/web-console/e2e/testcafe/components/FormField.js b/modules/web-console/e2e/testcafe/components/FormField.js index b0e89675103da..0c6c4ebab54f5 100644 --- a/modules/web-console/e2e/testcafe/components/FormField.js +++ b/modules/web-console/e2e/testcafe/components/FormField.js @@ -32,13 +32,12 @@ export class FormField { if (id) this._selector = Selector(`#${id}`).parent(this.constructor.ROOT_SELECTOR); else if (label) { - this._selector = Selector(() => { - return Array - .from(window.document.querySelectorAll(this.constructor.LABEL_SELECTOR)) - .filter((el) => el.textContent.contains(label)) - .map((el) => el.parent(this.constructor.ROOT_SELECTOR)) + this._selector = Selector((LABEL_SELECTOR, ROOT_SELECTOR, label) => { + return [].slice.call((window.document.querySelectorAll(LABEL_SELECTOR))) + .filter((el) => el.textContent.includes(label)) + .map((el) => el.closest(ROOT_SELECTOR)) .pop(); - }); + })(this.constructor.LABEL_SELECTOR, this.constructor.ROOT_SELECTOR, label); } else if (model) this._selector = AngularJSSelector.byModel(model).parent(this.constructor.ROOT_SELECTOR); diff --git a/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js b/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js index 7663e9d0037fc..7f2f1b15a10d1 100644 --- a/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js +++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js @@ -28,6 +28,8 @@ export default class PCFormFieldSizeController { onScaleChange; /** @type {ng.IFormController} */ innerForm; + /** @type {boolean?} */ + autofocus; static $inject = ['$element', '$attrs']; @@ -38,6 +40,10 @@ export default class PCFormFieldSizeController { {label: 'Mb', value: 1024 * 1024}, {label: 'Gb', value: 1024 * 1024 * 1024} ], + gigabytes: [ + {label: 'Gb', value: 1}, + {label: 'Tb', value: 1024} + ], seconds: [ {label: 'ns', value: 1 / 1000}, {label: 'ms', value: 1}, diff --git a/modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js b/modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js index 5e08df2921194..e7da7d5df0acd 100644 --- a/modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js +++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js @@ -36,6 +36,7 @@ export default { sizeType: '@?', sizeScaleLabel: '@?', onScaleChange: '&?', - ngDisabled: ' Date: Tue, 20 Nov 2018 14:13:48 +0700 Subject: [PATCH 144/403] IGNITE-9946 Web Console: Fixed fields validation order. (cherry picked from commit 7da186455f6fad9afd62be7f5802a4ee30667ef6) --- .../frontend/app/primitives/datepicker/index.pug | 6 +++--- .../frontend/app/primitives/form-field/dropdown.pug | 6 +++--- .../frontend/app/primitives/form-field/email.pug | 6 +++--- .../frontend/app/primitives/form-field/number.pug | 6 +++--- .../frontend/app/primitives/form-field/password.pug | 8 ++++---- .../frontend/app/primitives/form-field/phone.pug | 5 +++-- .../frontend/app/primitives/form-field/text.pug | 6 +++--- .../frontend/app/primitives/form-field/typeahead.pug | 6 +++--- .../frontend/app/primitives/timepicker/index.pug | 6 +++--- 9 files changed, 28 insertions(+), 27 deletions(-) diff --git a/modules/web-console/frontend/app/primitives/datepicker/index.pug b/modules/web-console/frontend/app/primitives/datepicker/index.pug index 28cd1a06c24c2..dd586dc7d39b8 100644 --- a/modules/web-console/frontend/app/primitives/datepicker/index.pug +++ b/modules/web-console/frontend/app/primitives/datepicker/index.pug @@ -58,8 +58,8 @@ mixin form-field__datepicker({ label, model, name, mindate, maxdate, minview = 1 ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/form-field/dropdown.pug b/modules/web-console/frontend/app/primitives/form-field/dropdown.pug index 96d8482cfc8af..b13a17749b19f 100644 --- a/modules/web-console/frontend/app/primitives/form-field/dropdown.pug +++ b/modules/web-console/frontend/app/primitives/form-field/dropdown.pug @@ -50,8 +50,8 @@ mixin form-field__dropdown({ label, model, name, disabled, required, multiple, p ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/form-field/email.pug b/modules/web-console/frontend/app/primitives/form-field/email.pug index 5fb0c3019fc67..b1e120254cdd3 100644 --- a/modules/web-console/frontend/app/primitives/form-field/email.pug +++ b/modules/web-console/frontend/app/primitives/form-field/email.pug @@ -29,10 +29,10 @@ mixin form-field__email({ label, model, name, disabled, required, placeholder, t ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) +form-field__error({ error: 'email', message: `${errLbl} has invalid format!` }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/form-field/number.pug b/modules/web-console/frontend/app/primitives/form-field/number.pug index ea907090e69d2..dfcfff36937bd 100644 --- a/modules/web-console/frontend/app/primitives/form-field/number.pug +++ b/modules/web-console/frontend/app/primitives/form-field/number.pug @@ -33,9 +33,6 @@ mixin form-field__number({ label, model, name, disabled, required, placeholder, ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) @@ -46,3 +43,6 @@ mixin form-field__number({ label, model, name, disabled, required, placeholder, +form-field__error({ error: 'step', message: `${errLbl} step should be ${step || 1}` }) +form-field__error({ error: 'number', message: 'Only numbers allowed' }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/form-field/password.pug b/modules/web-console/frontend/app/primitives/form-field/password.pug index 374d768213639..5103281fd7c50 100644 --- a/modules/web-console/frontend/app/primitives/form-field/password.pug +++ b/modules/web-console/frontend/app/primitives/form-field/password.pug @@ -44,10 +44,10 @@ mixin form-field__password({ label, model, name, disabled, required, placeholder ($input2.$dirty || $input2.$touched || $input2.$submitted) && $input2.$invalid ` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) - + +form-field__error({ error: 'mismatch', message: `Password does not match the confirm password!` }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/form-field/phone.pug b/modules/web-console/frontend/app/primitives/form-field/phone.pug index 9b60a40457b0b..4c794772dc621 100644 --- a/modules/web-console/frontend/app/primitives/form-field/phone.pug +++ b/modules/web-console/frontend/app/primitives/form-field/phone.pug @@ -29,8 +29,9 @@ mixin form-field__phone({ label, model, name, disabled, required, optional, plac ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) + if required + +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) + if block block - if required - +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) diff --git a/modules/web-console/frontend/app/primitives/form-field/text.pug b/modules/web-console/frontend/app/primitives/form-field/text.pug index ab8d14d2cbd26..5a2595ecc4c79 100644 --- a/modules/web-console/frontend/app/primitives/form-field/text.pug +++ b/modules/web-console/frontend/app/primitives/form-field/text.pug @@ -29,8 +29,8 @@ mixin form-field__text({ label, model, name, disabled, required, placeholder, ti ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/form-field/typeahead.pug b/modules/web-console/frontend/app/primitives/form-field/typeahead.pug index b2c62ae3d464e..ee2312f56366d 100644 --- a/modules/web-console/frontend/app/primitives/form-field/typeahead.pug +++ b/modules/web-console/frontend/app/primitives/form-field/typeahead.pug @@ -48,8 +48,8 @@ mixin form-field__typeahead({ label, model, name, disabled, required, placeholde ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) + + if block + block diff --git a/modules/web-console/frontend/app/primitives/timepicker/index.pug b/modules/web-console/frontend/app/primitives/timepicker/index.pug index f97a5a2ce956a..9a4dd97bfb394 100644 --- a/modules/web-console/frontend/app/primitives/timepicker/index.pug +++ b/modules/web-console/frontend/app/primitives/timepicker/index.pug @@ -56,8 +56,8 @@ mixin form-field__timepicker({ label, model, name, mindate, maxdate, disabled, r ng-messages=`$input.$error` ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid` ) - if block - block - if required +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` }) + + if block + block From 51366b4f37c8cdd4df2323687d8d5335b9a79a17 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Tue, 20 Nov 2018 16:30:35 +0700 Subject: [PATCH 145/403] IGNITE-9554 Web Agent: Exclude caches created via SQL from random data population in "Demo" mode. (cherry picked from commit 4fb1fc37a5016fb0386cb654cc527b685cb78349) --- .../service/DemoRandomCacheLoadService.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java index e6242fcc91d37..5b73e9660b9a0 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoRandomCacheLoadService.java @@ -29,6 +29,8 @@ import org.apache.ignite.services.Service; import org.apache.ignite.services.ServiceContext; +import static org.apache.ignite.internal.processors.query.QueryUtils.DFLT_SCHEMA; + /** * Demo service. Create cache and populate it by random int pairs. */ @@ -78,19 +80,19 @@ public DemoRandomCacheLoadService(int cnt) { @Override public void run() { try { for (String cacheName : ignite.cacheNames()) { - if (!DemoCachesLoadService.DEMO_CACHES.contains(cacheName)) { - IgniteCache cache = ignite.cache(cacheName); + IgniteCache cache = ignite.cache(cacheName); - if (cache != null) { - for (int i = 0, n = 1; i < cnt; i++, n++) { - Integer key = rnd.nextInt(RND_CNT); - Integer val = rnd.nextInt(RND_CNT); + if (cache != null && + !DemoCachesLoadService.DEMO_CACHES.contains(cacheName) && + !DFLT_SCHEMA.equalsIgnoreCase(cache.getConfiguration(CacheConfiguration.class).getSqlSchema())) { + for (int i = 0, n = 1; i < cnt; i++, n++) { + Integer key = rnd.nextInt(RND_CNT); + Integer val = rnd.nextInt(RND_CNT); - cache.put(key, val); + cache.put(key, val); - if (rnd.nextInt(100) < 30) - cache.remove(key); - } + if (rnd.nextInt(100) < 30) + cache.remove(key); } } } From c5d21e089636673ad66ed8b8b8690ef3d12aff22 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Tue, 20 Nov 2018 17:34:38 +0700 Subject: [PATCH 146/403] IGNITE-9811 Web Console: Do not block "Save" button in case some inputs are invalid and show validation when clicked. (cherry picked from commit 070819a528dfe944f39cee8f08331891820242f7) --- .../components/form-field-size/controller.js | 19 ++++++++++++++++- .../components/form-field-size/template.pug | 1 + .../showValidationError.directive.js | 21 ++++++++++++++++--- .../page-configure/components/pcValidation.js | 18 +++++++++++++++- .../app/components/page-profile/controller.js | 12 +++++++++-- .../app/components/page-profile/template.pug | 5 +---- .../app/components/page-signin/controller.ts | 3 +++ 7 files changed, 68 insertions(+), 11 deletions(-) diff --git a/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js b/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js index 7f2f1b15a10d1..e158c0af9ae9e 100644 --- a/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js +++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js @@ -67,7 +67,8 @@ export default class PCFormFieldSizeController { } $onDestroy() { - this.$element = null; + delete this.$element[0].focus; + this.$element = this.inputElement = null; } $onInit() { @@ -84,6 +85,8 @@ export default class PCFormFieldSizeController { this.ngModel.$validators.max = (value) => this.ngModel.$isEmpty(value) || value === void 0 || value <= this.max; this.ngModel.$validators.step = (value) => this.ngModel.$isEmpty(value) || value === void 0 || Math.floor(value) === value; + this.inputElement = this.$element[0].querySelector('input'); + this.$element[0].focus = () => this.inputElement.focus(); } $onChanges(changes) { @@ -143,4 +146,18 @@ export default class PCFormFieldSizeController { this.sizesMenu = PCFormFieldSizeController.sizeTypes.bytes; this.sizeScale = this.chooseSizeScale(); } + + notifyAboutError() { + if (this.$element) + this.$element.find('.form-field__error [bs-tooltip]').trigger('mouseenter'); + } + + hideError() { + if (this.$element) + this.$element.find('.form-field__error [bs-tooltip]').trigger('mouseleave'); + } + + triggerBlur() { + this.$element[0].dispatchEvent(new FocusEvent('blur', {relatedTarget: this.inputElement})); + } } diff --git a/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug b/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug index 147440b1bb0c5..6a2974d174d2a 100644 --- a/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug +++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug @@ -40,6 +40,7 @@ include /app/helpers/jade/mixins ng-required='$ctrl.required' ng-disabled='$ctrl.ngDisabled' ignite-form-field-input-autofocus='{{$ctrl.autofocus}}' + ng-on-blur='$ctrl.triggerBlur()' ) button.select-toggle( bs-select diff --git a/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js b/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js index 8720c56a173a7..31a8f2d06c091 100644 --- a/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js +++ b/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js @@ -21,8 +21,17 @@ */ export function directive($timeout) { return { - require: ['ngModel', '?^^bsCollapseTarget', '?^^igniteFormField', '?^^panelCollapsible'], - link(scope, el, attr, [ngModel, bsCollapseTarget, igniteFormField, panelCollapsible]) { + require: ['ngModel', '?^^bsCollapseTarget', '?^^igniteFormField', '?formFieldSize', '?^^panelCollapsible'], + link(scope, el, attr, [ngModel, bsCollapseTarget, igniteFormField, formFieldSize, panelCollapsible]) { + const formFieldController = igniteFormField || formFieldSize; + + let onBlur; + + scope.$on('$destroy', () => { + el[0].removeEventListener('blur', onBlur); + onBlur = null; + }); + const off = scope.$on('$showValidationError', (e, target) => { if (target !== ngModel) return; @@ -32,6 +41,12 @@ export function directive($timeout) { bsCollapseTarget && bsCollapseTarget.open(); panelCollapsible && panelCollapsible.open(); + if (!onBlur && formFieldController) { + onBlur = () => formFieldController.hideError(); + + el[0].addEventListener('blur', onBlur, {passive: true}); + } + $timeout(() => { if (el[0].scrollIntoViewIfNeeded) el[0].scrollIntoViewIfNeeded(); @@ -41,7 +56,7 @@ export function directive($timeout) { if (!attr.bsSelect) $timeout(() => el[0].focus()); - igniteFormField && igniteFormField.notifyAboutError(); + formFieldController && formFieldController.notifyAboutError(); }); }); } diff --git a/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js b/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js index 1702c35ae9ee5..697e9e955aad0 100644 --- a/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js +++ b/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js @@ -21,21 +21,37 @@ export class IgniteFormField { static animName = 'ignite-form-field__error-blink'; static eventName = 'webkitAnimationEnd oAnimationEnd msAnimationEnd animationend'; static $inject = ['$element', '$scope']; + constructor($element, $scope) { Object.assign(this, {$element}); this.$scope = $scope; } + $postLink() { this.onAnimEnd = () => this.$element.removeClass(IgniteFormField.animName); this.$element.on(IgniteFormField.eventName, this.onAnimEnd); } + $onDestroy() { this.$element.off(IgniteFormField.eventName, this.onAnimEnd); this.$element = this.onAnimEnd = null; } + notifyAboutError() { - if (this.$element) this.$element.addClass(IgniteFormField.animName); + if (!this.$element) + return; + + this.$element.addClass(IgniteFormField.animName); + this.$element.find('.form-field__error [bs-tooltip]').trigger('mouseenter'); } + + hideError() { + if (!this.$element) + return; + + this.$element.find('.form-field__error [bs-tooltip]').trigger('mouseleave'); + } + /** * Exposes control in $scope * @param {ng.INgModelController} control diff --git a/modules/web-console/frontend/app/components/page-profile/controller.js b/modules/web-console/frontend/app/components/page-profile/controller.js index 8586656743991..a2f344e98b9fc 100644 --- a/modules/web-console/frontend/app/components/page-profile/controller.js +++ b/modules/web-console/frontend/app/components/page-profile/controller.js @@ -19,7 +19,7 @@ import _ from 'lodash'; export default class PageProfileController { static $inject = [ - '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteCountries', 'User' + '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteCountries', 'User', 'IgniteFormUtils' ]; /** @@ -32,8 +32,9 @@ export default class PageProfileController { * @param {import('app/services/Confirm.service').Confirm} Confirm * @param {ReturnType} Countries * @param {ReturnType} User + * @param {ReturnType} FormUtils */ - constructor($root, $scope, $http, LegacyUtils, Messages, Focus, Confirm, Countries, User) { + constructor($root, $scope, $http, LegacyUtils, Messages, Focus, Confirm, Countries, User, FormUtils) { this.$root = $root; this.$scope = $scope; this.$http = $http; @@ -43,6 +44,7 @@ export default class PageProfileController { this.Confirm = Confirm; this.Countries = Countries; this.User = User; + this.FormUtils = FormUtils; } $onInit() { @@ -69,6 +71,12 @@ export default class PageProfileController { } saveUser() { + if (this.form.$invalid) { + this.FormUtils.triggerValidation(this.form); + + return; + } + return this.$http.post('/api/v1/profile/save', this.ui.user) .then(this.User.load) .then(() => { diff --git a/modules/web-console/frontend/app/components/page-profile/template.pug b/modules/web-console/frontend/app/components/page-profile/template.pug index 3c5fb52c980ba..68c321f44f5ab 100644 --- a/modules/web-console/frontend/app/components/page-profile/template.pug +++ b/modules/web-console/frontend/app/components/page-profile/template.pug @@ -154,9 +154,6 @@ div footer a.btn-ignite.btn-ignite--link-success(type='button' ui-sref='default-state') Cancel - button.btn-ignite.btn-ignite--success( - ng-click='$ctrl.saveUser()' - ng-disabled='$ctrl.form.$invalid' - ) + button.btn-ignite.btn-ignite--success(ng-click='$ctrl.saveUser()') svg.icon-left(ignite-icon='checkmark') | Save Changes diff --git a/modules/web-console/frontend/app/components/page-signin/controller.ts b/modules/web-console/frontend/app/components/page-signin/controller.ts index 724da41dadeb9..c461b86e92f0d 100644 --- a/modules/web-console/frontend/app/components/page-signin/controller.ts +++ b/modules/web-console/frontend/app/components/page-signin/controller.ts @@ -66,7 +66,10 @@ export default class { return this.Auth.signin(this.data.email, this.data.password).catch((res) => { this.IgniteMessages.showError(null, res.data); + this.setServerError(res.data); + + this.IgniteFormUtils.triggerValidation(this.form); }); } } From 640119e3c7454c2fc775561d127886707a0d7a98 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Tue, 20 Nov 2018 23:56:22 +0700 Subject: [PATCH 147/403] IGNITE-10349 Web Console: Added check for supported MongoDb version. (cherry picked from commit 8bd59ccfff778ba59d55707c5182c09e391c0103) --- modules/web-console/backend/index.js | 10 +++++----- modules/web-console/backend/launch-tools.js | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/modules/web-console/backend/index.js b/modules/web-console/backend/index.js index 4697de91eaffd..c601c0bf8a121 100644 --- a/modules/web-console/backend/index.js +++ b/modules/web-console/backend/index.js @@ -23,17 +23,17 @@ const appPath = require('app-module-path'); appPath.addPath(__dirname); appPath.addPath(path.join(__dirname, 'node_modules')); -const { migrate, init } = require('./launch-tools'); +const { checkMongo, migrate, init } = require('./launch-tools'); const injector = require('./injector'); injector.log.info = () => {}; injector.log.debug = () => {}; -Promise.all([injector('settings'), injector('mongo')]) - .then(([{mongoUrl}]) => { - return migrate(mongoUrl, 'Ignite', path.join(__dirname, 'migrations')); - }) +injector('mongo') + .then(() => checkMongo()) + .then(() => injector('settings')) + .then(({mongoUrl}) => migrate(mongoUrl, 'Ignite', path.join(__dirname, 'migrations'))) .then(() => Promise.all([injector('settings'), injector('api-server'), injector('agents-handler'), injector('browsers-handler')])) .then(init) .catch((err) => { diff --git a/modules/web-console/backend/launch-tools.js b/modules/web-console/backend/launch-tools.js index f1f3b2f4e835c..3870b0d6c754f 100644 --- a/modules/web-console/backend/launch-tools.js +++ b/modules/web-console/backend/launch-tools.js @@ -21,6 +21,7 @@ const _ = require('lodash'); const http = require('http'); const https = require('https'); const MigrateMongoose = require('migrate-mongoose'); +const mongoose = require('mongoose'); /** * Event listener for HTTP server "error" event. @@ -106,4 +107,21 @@ const migrate = (dbConnectionUri, group, migrationsPath, collectionName) => { }); }; -module.exports = { migrate, init }; +/** + * Check version of used MongoDB. + */ +const checkMongo = () => { + const versionValid = (mijor, minor) => mijor === 3 && minor >= 2 && minor <= 4; + + const admin = new mongoose.mongo.Admin(mongoose.connection.db, null, global.Promise); + + return admin.buildInfo() + .then((info) => { + const versions = info.version.split('.'); + + if (!versionValid(parseInt(versions[0]), parseInt(versions[1]))) + throw Error(`Unsupported version of MongoDB ${info.version}. Supported versions: 3.2.x-3.4.x`); + }); +}; + +module.exports = { checkMongo, migrate, init }; From 3c13aceb875c21b3f38847a470f8615def401721 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Wed, 21 Nov 2018 00:17:58 +0700 Subject: [PATCH 148/403] IGNITE-10351 Web Console: Added new fields sysWorkerBlockedTimeout and checkpointReadLockTimeout to "Configuration" screen. (cherry picked from commit 296737a1b63c42a4fa1f398a881872c9f3b21b05) --- modules/web-console/backend/app/schemas.js | 4 +++- .../cluster-edit-form/templates/data-storage.pug | 9 +++++++++ .../cluster-edit-form/templates/failover.pug | 9 +++++++++ .../generator/ConfigurationGenerator.js | 13 +++++++++++-- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/modules/web-console/backend/app/schemas.js b/modules/web-console/backend/app/schemas.js index cf174f7ad3086..69d4a4cce81d6 100644 --- a/modules/web-console/backend/app/schemas.js +++ b/modules/web-console/backend/app/schemas.js @@ -985,6 +985,7 @@ module.exports.factory = function(mongoose) { consistentId: String, failureDetectionTimeout: Number, clientFailureDetectionTimeout: Number, + systemWorkerBlockedTimeout: Number, workDirectory: String, lateAffinityAssignment: Boolean, utilityCacheKeepAliveTime: Number, @@ -1054,7 +1055,8 @@ module.exports.factory = function(mongoose) { fileIOFactory: {type: String, enum: ['RANDOM', 'ASYNC']}, walAutoArchiveAfterInactivity: Number, writeThrottlingEnabled: Boolean, - walCompactionEnabled: Boolean + walCompactionEnabled: Boolean, + checkpointReadLockTimeout: Number }, memoryConfiguration: { systemCacheInitialSize: Number, diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug index e18b0cbabaf78..8f9a33ebfe259 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug @@ -256,6 +256,15 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo min: '1', tip: 'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store' }) + .pc-form-grid-col-60(ng-if='$ctrl.available("2.7.0")') + +form-field__number({ + label: 'Checkpoint read lock timeout:', + model: `${model}.checkpointReadLockTimeout`, + name: '"DataStorageCheckpointReadLockTimeout"', + placeholder: 'System workers blocked timeout', + min: '1', + tip: 'Timeout for checkpoint read lock acquisition' + }) .pc-form-grid-col-20 +form-field__number({ label: 'Checkpoint threads:', diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug index 3fafe9f86f845..27797ed1b0895 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug @@ -47,6 +47,15 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`) min: '1', tip: 'Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed' }) + .pc-form-grid-col-60(ng-if='$ctrl.available("2.7.0")') + +form-field__number({ + label: 'System workers blocked timeout:', + model: model + '.systemWorkerBlockedTimeout', + name: '"SystemWorkerBlockedTimeout"', + placeholder: 'Failure detection timeout', + min: '1', + tip: 'Maximum inactivity period for system worker' + }) .pc-form-grid-col-60 mixin clusters-failover-spi diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js index d80967e2cfcdc..6fba32a9eb253 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js @@ -1131,6 +1131,9 @@ export default class IgniteConfigurationGenerator { if (available('2.0.0')) { cfg.longProperty('failureDetectionTimeout') .longProperty('clientFailureDetectionTimeout'); + + if (available('2.7.0')) + cfg.longProperty('systemWorkerBlockedTimeout'); } _.forEach(cluster.failoverSpi, (spi) => { @@ -1465,8 +1468,14 @@ export default class IgniteConfigurationGenerator { storageBean.varArgProperty('dataRegionConfigurations', 'dataRegionConfigurations', dataRegionCfgs, 'org.apache.ignite.configuration.DataRegionConfiguration'); storageBean.stringProperty('storagePath') - .longProperty('checkpointFrequency') - .intProperty('checkpointThreads') + .longProperty('checkpointFrequency'); + + if (available('2.7.0')) { + storageBean + .longProperty('checkpointReadLockTimeout'); + } + + storageBean.intProperty('checkpointThreads') .enumProperty('checkpointWriteOrder') .enumProperty('walMode') .stringProperty('walPath') From 9c1c7f4a39df239f717d0fafdefc5c907eba8487 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 21 Nov 2018 18:48:39 +0700 Subject: [PATCH 149/403] IGNITE-9923 Fixed unused imports. --- .../ignite/console/agent/db/dialect/OracleMetadataDialect.java | 1 - .../ignite/console/demo/service/DemoComputeLoadService.java | 1 - .../org/apache/ignite/console/demo/task/DemoComputeTask.java | 1 - 3 files changed, 3 deletions(-) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java index b8b72d5422b94..f7c183f2461d7 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java @@ -39,7 +39,6 @@ import static java.sql.Types.BIGINT; import static java.sql.Types.BLOB; -import static java.sql.Types.BOOLEAN; import static java.sql.Types.CHAR; import static java.sql.Types.CLOB; import static java.sql.Types.DATE; diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java index e775a9a6dd337..3c767c85dac91 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java @@ -17,7 +17,6 @@ package org.apache.ignite.console.demo.service; -import java.util.Random; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.ignite.Ignite; diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java index 14f2508d82e9b..7dba1b73465c8 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Random; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobAdapter; From 6c45e5b016d98146cdcddf114da4b6968d9a96f4 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 15 Oct 2018 09:39:21 +0700 Subject: [PATCH 150/403] IGNITE-9862 Web Console: Add support for "date", "time" and "date-and-time" to InputDialog. --- .../input-dialog/input-dialog.service.js | 119 ------------- .../input-dialog/input-dialog.service.ts | 162 ++++++++++++++++++ .../input-dialog/input-dialog.tpl.pug | 35 ++++ .../app/primitives/datepicker/index.scss | 1 + .../app/primitives/timepicker/index.scss | 1 + 5 files changed, 199 insertions(+), 119 deletions(-) delete mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js create mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js deleted file mode 100644 index 6ddf051290e13..0000000000000 --- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import _ from 'lodash'; -import controller from './input-dialog.controller'; -import templateUrl from './input-dialog.tpl.pug'; -import {CancellationError} from 'app/errors/CancellationError'; - -export default class InputDialog { - static $inject = ['$modal', '$q']; - - /** - * @param {mgcrea.ngStrap.modal.IModalService} $modal - * @param {ng.IQService} $q - */ - constructor($modal, $q) { - this.$modal = $modal; - this.$q = $q; - } - - /** - * Fabric for creating modal instance with different input types. - * - * @param {Object} args Options for rendering inputs: - * @param {'text'|'number'} args.mode Input type. - * @param {String} args.title Dialog title. - * @param {String} args.label Input field label. - * @param {String} args.tip Message for tooltip in label. - * @param {String|Number} args.value Default value. - * @param {String} args.placeholder Placeholder for input. - * @param {Function} [args.toValidValue] Validator function. - * @param {Number} args.min Min value for number input. - * @param {Number} args.max Max value for number input. - * @param {String} args.postfix Postfix for units in numer input. - * @return {Promise.} User input. - */ - dialogFabric(args) { - const deferred = this.$q.defer(); - - const modal = this.$modal({ - templateUrl, - resolve: { - deferred: () => deferred, - ui: () => args - }, - controller, - controllerAs: 'ctrl' - }); - - const modalHide = modal.hide; - - modal.hide = () => deferred.reject(new CancellationError()); - - return deferred.promise - .finally(modalHide); - } - - /** - * Open input dialog to configure custom value. - * - * @param {String} title Dialog title. - * @param {String} label Input field label. - * @param {String} value Default value. - * @param {Function} [toValidValue] Validator function. - * @param {'text'|'number'} mode Input type. - * @returns {ng.IPromise} User input. - */ - input(title, label, value, toValidValue, mode = 'text') { - return this.dialogFabric({title, label, value, toValidValue, mode}); - } - - /** - * Open input dialog to configure cloned object name. - * - * @param {String} srcName Name of source object. - * @param {Array.} names List of already exist names. - * @returns {ng.IPromise} New name. - */ - clone(srcName, names) { - const uniqueName = (value) => { - let num = 1; - let tmpName = value; - - while (_.includes(names, tmpName)) { - tmpName = `${value}_${num}`; - - num++; - } - - return tmpName; - }; - - return this.input('Clone', 'New name', uniqueName(srcName), uniqueName); - } - - /** - * Open input dialog to configure custom number value. - * - * @param {Object} options Object with settings for rendering number input. - * @returns {Promise.} User input. - */ - number(options) { - return this.dialogFabric({mode: 'number', ...options}); - } -} diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts new file mode 100644 index 0000000000000..4d8ebd7459827 --- /dev/null +++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import _ from 'lodash'; +import controller from './input-dialog.controller'; +import templateUrl from './input-dialog.tpl.pug'; +import {CancellationError} from 'app/errors/CancellationError'; + +type InputModes = 'text' | 'number' | 'date' | 'time' | 'date-and-time'; + +interface ValidationFunction { + (value: T): boolean +} + +/** + * Options for rendering inputs. + */ +interface InputOptions { + /** Input type. */ + mode?: InputModes, + /** Dialog title. */ + title?: string, + /** Input field label. */ + label?: string, + /** Message for tooltip in label. */ + tip?: string, + /** Default value. */ + value: T, + /** Placeholder for input. */ + placeholder?: string, + /** Validator function. */ + toValidValue?: ValidationFunction, + /** Min value for number input. */ + min?: number, + /** Max value for number input. */ + max?: number, + /** Postfix for units in number input. */ + postfix?: string +} + +export default class InputDialog { + static $inject = ['$modal', '$q']; + + constructor(private $modal: mgcrea.ngStrap.modal.IModalService, private $q: ng.IQService) {} + + /** + * Fabric for creating modal instance with different input types. + * + * @returns User input. + */ + private dialogFabric(args: InputOptions) { + const deferred = this.$q.defer(); + + const modal = this.$modal({ + templateUrl, + resolve: { + deferred: () => deferred, + ui: () => args + }, + controller, + controllerAs: 'ctrl' + }); + + const modalHide = modal.hide; + + modal.hide = () => deferred.reject(new CancellationError()); + + return deferred.promise + .finally(modalHide); + } + + /** + * Open input dialog to configure custom value. + * + * @param title Dialog title. + * @param label Input field label. + * @param value Default value. + * @param toValidValue Validator function. + * @param mode Input type. + */ + input(title: string, label: string, value: T, toValidValue?: ValidationFunction, mode: InputModes = 'text') { + return this.dialogFabric({title, label, value, toValidValue, mode}); + } + + /** + * Open input dialog to configure cloned object name. + * + * @param srcName Name of source object. + * @param names List of already exist names. + * @returns New name. + */ + clone(srcName: string, names: Array) { + const uniqueName = (value) => { + let num = 1; + let tmpName = value; + + while (_.includes(names, tmpName)) { + tmpName = `${value}_${num}`; + + num++; + } + + return tmpName; + }; + + return this.input('Clone', 'New name', uniqueName(srcName), uniqueName); + } + + /** + * Open input dialog to configure custom number value. + * + * @param options Object with settings for rendering number input. + * @returns User input. + */ + number(options: InputOptions) { + return this.dialogFabric({mode: 'number', ...options}); + } + + /** + * Open input dialog to configure custom date value. + * + * @param options Settings for rendering date input. + * @returns User input. + */ + date(options: InputOptions) { + return this.dialogFabric({mode: 'date', ...options}); + } + + /** + * Open input dialog to configure custom time value. + * + * @param options Settings for rendering time input. + * @returns User input. + */ + time(options: InputOptions) { + return this.dialogFabric({mode: 'time', ...options}); + } + + /** + * Open input dialog to configure custom date and time value. + * + * @param options Settings for rendering date and time inputs. + * @returns User input. + */ + dateTime(options: InputOptions) { + return this.dialogFabric({mode: 'date-and-time', ...options}); + } +} diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug index 95cb059ba6ff9..0d0d5ca242cb1 100644 --- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug +++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug @@ -53,6 +53,41 @@ include /app/helpers/jade/mixins required: true }) + .row(ng-switch-when='date') + .col-100 + .form-field--inline + +form-field__datepicker({ + label: '{{ ctrl.options.label }}', + model: 'ctrl.options.value', + name: '"date"', + minview: 0, + format: 'dd/MM/yyyy' + }) + + .row(ng-switch-when='time') + .col-100 + .form-field--inline + +form-field__timepicker({ + label: '{{ ctrl.options.label }}', + model: 'ctrl.options.value', + name: '"time"' + }) + + .row(ng-switch-when='date-and-time') + .col-100 + .form-field--inline + +form-field__datepicker({ + label: '{{ ctrl.options.label }}', + model: 'ctrl.options.value', + name: '"date"', + minview: 0, + format: 'dd/MM/yyyy' + }) + .form-field--inline + +form-field__timepicker({ + model: 'ctrl.options.value', + name: '"time"' + }) .modal-footer div button#copy-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='$hide()') Cancel diff --git a/modules/web-console/frontend/app/primitives/datepicker/index.scss b/modules/web-console/frontend/app/primitives/datepicker/index.scss index c8edab58f3558..c527a42dcca91 100644 --- a/modules/web-console/frontend/app/primitives/datepicker/index.scss +++ b/modules/web-console/frontend/app/primitives/datepicker/index.scss @@ -18,6 +18,7 @@ .datepicker.dropdown-menu { width: 250px; height: 270px; + z-index: 2000; button { outline: none; diff --git a/modules/web-console/frontend/app/primitives/timepicker/index.scss b/modules/web-console/frontend/app/primitives/timepicker/index.scss index 643c741b5f8f3..1b21c707239c2 100644 --- a/modules/web-console/frontend/app/primitives/timepicker/index.scss +++ b/modules/web-console/frontend/app/primitives/timepicker/index.scss @@ -18,6 +18,7 @@ .timepicker.dropdown-menu { padding: 0 4px; line-height: 30px; + z-index: 2000; button { outline: none; From 8e854edae6f4daae0c6084c2ee8886429f364b09 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 17 Oct 2018 15:43:52 +0700 Subject: [PATCH 151/403] IGNITE-9862 Minor fixes. --- .../frontend/app/components/input-dialog/input-dialog.tpl.pug | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug index 0d0d5ca242cb1..4674c97db4be4 100644 --- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug +++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug @@ -86,6 +86,7 @@ include /app/helpers/jade/mixins .form-field--inline +form-field__timepicker({ model: 'ctrl.options.value', + format: '{{ ctrl.options.format }}', name: '"time"' }) .modal-footer From 253640e83757bbab0b14a8855ba7a3c19c01dcd5 Mon Sep 17 00:00:00 2001 From: Ilya Murchenko Date: Wed, 31 Oct 2018 18:36:22 +0700 Subject: [PATCH 152/403] IGNITE-9010: Move web-console build to dedicated root directory - Fixes #4528. (cherry picked from commit a1b7239d60f1e5fbf19e4fc9bb6db425e27ca2d0) --- .../web-console}/standalone/Dockerfile | 73 +++++++++++-------- docker/web-console/standalone/README.txt | 35 +++++++++ .../standalone/docker-entrypoint.sh | 0 .../web-console}/standalone/nginx/nginx.conf | 0 .../standalone/nginx/web-console.conf | 0 modules/web-console/DEVNOTES.txt | 13 ---- modules/web-console/pom.xml | 38 ---------- 7 files changed, 76 insertions(+), 83 deletions(-) rename {modules/web-console/docker => docker/web-console}/standalone/Dockerfile (53%) create mode 100644 docker/web-console/standalone/README.txt rename {modules/web-console/docker => docker/web-console}/standalone/docker-entrypoint.sh (100%) rename {modules/web-console/docker => docker/web-console}/standalone/nginx/nginx.conf (100%) rename {modules/web-console/docker => docker/web-console}/standalone/nginx/web-console.conf (100%) diff --git a/modules/web-console/docker/standalone/Dockerfile b/docker/web-console/standalone/Dockerfile similarity index 53% rename from modules/web-console/docker/standalone/Dockerfile rename to docker/web-console/standalone/Dockerfile index 9b007349b6a62..dfcb188b2dac6 100644 --- a/modules/web-console/docker/standalone/Dockerfile +++ b/docker/web-console/standalone/Dockerfile @@ -15,62 +15,71 @@ # limitations under the License. # + +#~~~~~~~~~~~~~~~~~~# +# Frontend build # +#~~~~~~~~~~~~~~~~~~# FROM node:8-slim as frontend-build ENV NPM_CONFIG_LOGLEVEL error WORKDIR /opt/web-console -# Install node modules for frontend. -COPY frontend/package*.json frontend/ -RUN (cd frontend && npm install --no-optional) - -# Copy source. +# Install node modules and build sources COPY frontend frontend +RUN cd frontend && \ + npm install --no-optional && \ + npm run build -RUN (cd frontend && npm run build) +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# +# Web Console Standalone assemble # +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# FROM node:8-slim ENV NPM_CONFIG_LOGLEVEL error -# Update package list & install. -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 \ - && echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/3.4 main" | tee /etc/apt/sources.list.d/mongodb-org-3.4.list - -# Update package list & install. -RUN apt-get update \ - && apt-get install -y nginx-light mongodb-org-server dos2unix \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install global node packages. +# Install global node packages RUN npm install -g pm2 -WORKDIR /opt/web-console - -COPY docker/standalone/docker-entrypoint.sh docker-entrypoint.sh +# Update software sources and install missing applications +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 && \ + echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/3.4 main" > /etc/apt/sources.list.d/mongodb-org-3.4.list && \ + apt update && \ + apt install -y --no-install-recommends \ + nginx-light \ + mongodb-org-server \ + dos2unix && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -RUN chmod +x docker-entrypoint.sh \ - && dos2unix docker-entrypoint.sh - -# Copy nginx config. -COPY docker/standalone/nginx/* /etc/nginx/ +WORKDIR /opt/web-console -# Install node modules for frontend and backend modules. +# Install node modules for backend COPY backend/package*.json backend/ -RUN (cd backend && npm install --no-optional --production) +RUN cd backend && \ + npm install --no-optional --production -# Copy source. +# Copy and build sources COPY backend backend +RUN cd backend && \ + npm run build -COPY web-agent/target/ignite-web-agent-*.zip backend/agent_dists +# Copy Ignite Web Agent module package +COPY ignite-web-agent-*.zip backend/agent_dists +# Copy previously built frontend COPY --from=frontend-build /opt/web-console/frontend/build static -VOLUME ["/etc/nginx"] -VOLUME ["/data/db"] -VOLUME ["/opt/web-console/serve/agent_dists"] +# Copy and fix entrypoint script +COPY docker-entrypoint.sh docker-entrypoint.sh +RUN chmod +x docker-entrypoint.sh \ + && dos2unix docker-entrypoint.sh + +# Copy nginx configuration +COPY nginx/* /etc/nginx/ EXPOSE 80 + ENTRYPOINT ["/opt/web-console/docker-entrypoint.sh"] + diff --git a/docker/web-console/standalone/README.txt b/docker/web-console/standalone/README.txt new file mode 100644 index 0000000000000..c97e7924fbdc8 --- /dev/null +++ b/docker/web-console/standalone/README.txt @@ -0,0 +1,35 @@ +Apache Ignite Web Console Standalone Docker module +================================================== +Apache Ignite Web Console Standalone Docker module provides Dockerfile and accompanying files +for building docker image of Web Console. + + +Ignite Web Console Standalone Docker Image Build Instructions +============================================================= +1) Build ignite-web-console module + + mvn clean install -P web-console -DskipTests -T 2C -pl :ignite-web-console -am + +2) Copy ignite-web-agent-.zip from 'modules/web-console/web-agent/target' + to 'docker/web-console/standalone' directory + + cp -rf modules/web-console/web-agent/target/ignite-web-agent-*.zip docker/web-console/standalone + +3) Go to Apache Ignite Web Console Docker module directory and copy Apache + Ignite Web Console's frontend and backend directory + + cd docker/web-console/standalone + cp -rf ../../../modules/web-console/backend ./ + cp -rf ../../../modules/web-console/frontend ./ + +4) Build docker image + + docker build . -t apacheignite/web-console-standalone:[:] + + Prepared image will be available in local docker registry (can be seen + issuing `docker images` command) + +5) Clean up + + rm -rf backend frontend ignite-web-agent* + diff --git a/modules/web-console/docker/standalone/docker-entrypoint.sh b/docker/web-console/standalone/docker-entrypoint.sh similarity index 100% rename from modules/web-console/docker/standalone/docker-entrypoint.sh rename to docker/web-console/standalone/docker-entrypoint.sh diff --git a/modules/web-console/docker/standalone/nginx/nginx.conf b/docker/web-console/standalone/nginx/nginx.conf similarity index 100% rename from modules/web-console/docker/standalone/nginx/nginx.conf rename to docker/web-console/standalone/nginx/nginx.conf diff --git a/modules/web-console/docker/standalone/nginx/web-console.conf b/docker/web-console/standalone/nginx/web-console.conf similarity index 100% rename from modules/web-console/docker/standalone/nginx/web-console.conf rename to docker/web-console/standalone/nginx/web-console.conf diff --git a/modules/web-console/DEVNOTES.txt b/modules/web-console/DEVNOTES.txt index a06608d14ad8a..e7c5eb3e92898 100644 --- a/modules/web-console/DEVNOTES.txt +++ b/modules/web-console/DEVNOTES.txt @@ -51,19 +51,6 @@ To build docker images from sources run following command in Ignite project root Prepared image can be listed with `docker images` command. -Ignite Web Console Standalone Docker Image Build Manual Instructions -==================================================================== -Install Docker (version >=17.05) using instructions from https://www.docker.com/community-edition. - -1. Build Apache Ignite Web Agent archive as described in `modules/web-console/web-agent/README.txt`. -2. Goto Web Console's module directory: `cd modules/web-console` -3. Build docker image: - -"docker build . -t apacheignite/web-console-standalone[:] -f docker/standalone/Dockerfile" - -Prepared image can be listed in `docker images` command output. - - Ignite Web Console Backend Docker Image Build Manual Instructions ==================================================================== Install Docker (version >=17.05) using instructions from https://www.docker.com/community-edition. diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml index 79293ff6fea94..273e5fe1f84d6 100644 --- a/modules/web-console/pom.xml +++ b/modules/web-console/pom.xml @@ -336,27 +336,6 @@ - - docker-build-standalone - package - - exec - - - docker - - build - -f - docker/standalone/Dockerfile - -t - - ${docker.registry.host}/${docker.repository}/${docker.standalone.image}:${project.version} - - ${project.basedir} - - - - docker-push-backend deploy @@ -390,23 +369,6 @@ - - - docker-push-standalone - deploy - - exec - - - docker - - push - - ${docker.registry.host}/${docker.repository}/${docker.standalone.image}:${project.version} - - - - From f22e5816305c604c727a78528867b459ed134c2d Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 21 Nov 2018 19:11:52 +0700 Subject: [PATCH 153/403] IGNITE-9941 Fixes after merge with master. --- .../input-dialog/input-dialog.service.ts | 12 +++++++++++- .../components/input-dialog/input-dialog.tpl.pug | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts index 4d8ebd7459827..b2a241fb9206a 100644 --- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts +++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts @@ -20,7 +20,7 @@ import controller from './input-dialog.controller'; import templateUrl from './input-dialog.tpl.pug'; import {CancellationError} from 'app/errors/CancellationError'; -type InputModes = 'text' | 'number' | 'date' | 'time' | 'date-and-time'; +type InputModes = 'text' | 'number' | 'email' | 'date' | 'time' | 'date-and-time'; interface ValidationFunction { (value: T): boolean @@ -130,6 +130,16 @@ export default class InputDialog { return this.dialogFabric({mode: 'number', ...options}); } + /** + * Open input dialog to configure custom e-mail. + * + * @param options Object with settings for rendering e-mail input. + * @return User input. + */ + email(options: InputOptions) { + return this.dialogFabric({mode: 'email', ...options}); + } + /** * Open input dialog to configure custom date value. * diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug index 4674c97db4be4..d79cb565e3294 100644 --- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug +++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug @@ -39,6 +39,7 @@ include /app/helpers/jade/mixins ignite-form-field-input-autofocus='true' ignite-on-enter='form.$valid && ctrl.confirm()' ) + .row(ng-switch-when='number') .col-100 +form-field__number({ @@ -53,6 +54,19 @@ include /app/helpers/jade/mixins required: true }) + .row(ng-switch-when='email') + .col-100 + +form-field__email({ + label: '{{ ctrl.options.label }}', + model: 'ctrl.options.value', + name: '"email"', + required: true, + placeholder: 'Input email' + })( + ignite-form-field-input-autofocus='true' + ignite-on-enter='form.$valid && ctrl.confirm()' + ) + .row(ng-switch-when='date') .col-100 .form-field--inline From 172030ed6e26f9143f040257748a6133d6a613d0 Mon Sep 17 00:00:00 2001 From: Max-Pudov Date: Wed, 21 Nov 2018 13:51:08 +0300 Subject: [PATCH 154/403] IGNITE-9872 [Test falied] IgniteSinkConnectorTest.testSinkPutsWithoutTransformation (updating Kafka to 2.0.1) - Fixes #5430. Signed-off-by: Dmitriy Pavlov (cherry picked from commit bbf8f30f224b70f1504a5cd76f230a6b73ed4fb3) --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index 4781d294f1922..338fd580715ea 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -89,7 +89,7 @@ 2.4_1 2.4 r938 - 1.1.1 + 2.0.1 4.0.2 2.11.0 7.4.0_1 From 0d6889f4868da3115292e45a80a043c63e681682 Mon Sep 17 00:00:00 2001 From: ipavlukhin Date: Tue, 6 Nov 2018 19:43:02 +0300 Subject: [PATCH 155/403] IGNITE-10103 Fixed IgnitePersistentStoreDataStructuresTest.testSequenceAfterAutoactivation - Fixes #5230. Signed-off-by: Alexey Goncharuk (cherry picked from commit 5459469c4f7bf1e89f4f7ee4145d1481c164725d) --- ...gnitePersistentStoreDataStructuresTest.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java index dc4e17e657b30..80fb0e077c35d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java @@ -22,6 +22,7 @@ import org.apache.ignite.IgniteAtomicLong; import org.apache.ignite.IgniteAtomicSequence; import org.apache.ignite.IgniteCountDownLatch; +import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLock; import org.apache.ignite.IgniteQueue; import org.apache.ignite.IgniteSemaphore; @@ -185,10 +186,19 @@ public void testSequenceAfterAutoactivation() throws Exception { final Ignite node = startGrids(2); - IgniteInternalFuture fut = GridTestUtils.runAsync(new Runnable() { - @Override public void run() { - // Should not hang. - node.atomicSequence(seqName, 0, false); + IgniteInternalFuture fut = GridTestUtils.runAsync(() -> { + while (true) { + try { + // Should not hang. + node.atomicSequence(seqName, 0, false); + + break; + } + catch (IgniteException e) { + // Can fail on not yet activated cluster. Retry until success. + assertTrue(e.getMessage() + .contains("Can not perform the operation because the cluster is inactive")); + } } }); From 8bed78b836d08e51840033ff3d7334d4b63cf49e Mon Sep 17 00:00:00 2001 From: ipavlukhin Date: Wed, 14 Nov 2018 15:16:58 +0300 Subject: [PATCH 156/403] IGNITE-7955: MVCC: IgniteCache.localPeek operation support. This closes #5284. (cherry picked from commit b85b041bcbf199b94a16c9f09dd9d2685f65cbc3) --- .../java/org/apache/ignite/IgniteCache.java | 4 +- .../processors/cache/GridCacheAdapter.java | 7 +- .../processors/cache/GridCacheEntryEx.java | 10 ++ .../processors/cache/GridCacheMapEntry.java | 21 +++ .../processors/cache/mvcc/MvccUtils.java | 8 +- .../cache/GridCacheTestEntryEx.java | 8 +- .../mvcc/CacheMvccOperationChecksTest.java | 8 - .../cache/mvcc/MvccCachePeekTest.java | 161 ++++++++++++++++++ .../testsuites/IgniteCacheMvccTestSuite.java | 41 +++++ 9 files changed, 248 insertions(+), 20 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachePeekTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java index 70ee0d56f5961..395c8f89a8692 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java @@ -441,9 +441,9 @@ public IgniteFuture localLoadCacheAsync(@Nullable IgniteBiPredicate public void localEvict(Collection keys); /** - * Peeks at in-memory cached value using default optional peek mode. + * Peeks at a value in the local storage using an optional peek mode. *

- * This method will not load value from any persistent store or from a remote node. + * This method will not load a value from the configured {@link CacheStore} or from a remote node. *

Transactions

* This method does not participate in any transactions. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index a62c64cb7c189..bd0d5835c289b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -823,9 +823,6 @@ else if (modes.heap) { ctx.checkSecurity(SecurityPermission.CACHE_READ); - //TODO IGNITE-7955 - MvccUtils.verifyMvccOperationSupport(ctx, "Peek"); - PeekModes modes = parsePeekModes(peekModes, false); KeyCacheObject cacheKey = ctx.toCacheKeyObject(key); @@ -897,7 +894,9 @@ else if (modes.heap) { ctx.shared().database().checkpointReadLock(); try { - cacheVal = e.peek(modes.heap, modes.offheap, topVer, plc); + cacheVal = ctx.mvccEnabled() + ? e.mvccPeek(modes.heap && !modes.offheap) + : e.peek(modes.heap, modes.offheap, topVer, plc); } catch (GridCacheEntryRemovedException ignore) { if (log.isDebugEnabled()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index cfd70ecc07604..319c134a85f88 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -712,6 +712,16 @@ public boolean tmLock(IgniteInternalTx tx, */ public boolean checkSerializableReadVersion(GridCacheVersion serReadVer) throws GridCacheEntryRemovedException; + /** + * Retrieves the last committed MVCC entry version. + * @param onheapOnly {@code True} if a specified peek mode instructs to look only in the on-heap storage. + * @return Last committed entry if either or {@code null} otherwise. + * @throws GridCacheEntryRemovedException If entry has been removed. + * @throws IgniteCheckedException If failed. + */ + @Nullable public CacheObject mvccPeek(boolean onheapOnly) + throws GridCacheEntryRemovedException, IgniteCheckedException; + /** * Peeks into entry without loading value or updating statistics. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index e09f3734c591f..a68a4d317e2db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -114,6 +114,7 @@ import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE; import static org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome.INVOKE_NO_OP; import static org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome.REMOVE_NO_VAL; +import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.MVCC_MAX_SNAPSHOT; import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.compareIgnoreOpCounter; import static org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.RowData.NO_KEY; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.DUPLICATE_KEY; @@ -3116,6 +3117,26 @@ int hash() { return hash; } + /** {@inheritDoc} */ + @Nullable @Override public CacheObject mvccPeek(boolean onheapOnly) + throws GridCacheEntryRemovedException, IgniteCheckedException { + if (onheapOnly) + return null; + + lockEntry(); + + try { + checkObsolete(); + + CacheDataRow row = cctx.offheap().mvccRead(cctx, key, MVCC_MAX_SNAPSHOT); + + return row != null ? row.value() : null; + } + finally { + unlockEntry(); + } + } + /** {@inheritDoc} */ @Nullable @Override public CacheObject peek( boolean heap, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java index 972d4d9b4bbc4..ffb062daf3a76 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java @@ -100,13 +100,13 @@ public class MvccUtils { /** */ public static final int MVCC_VISIBLE = 2; - /** */ + /** A special version visible by everyone */ public static final MvccVersion INITIAL_VERSION = mvccVersion(MVCC_CRD_START_CNTR, MVCC_INITIAL_CNTR, MVCC_START_OP_CNTR); - /** */ - public static final MvccVersion MVCC_VERSION_NA = - mvccVersion(MVCC_CRD_COUNTER_NA, MVCC_COUNTER_NA, MVCC_OP_COUNTER_NA); + /** A special snapshot for which all committed versions are visible */ + public static final MvccSnapshot MVCC_MAX_SNAPSHOT = + new MvccSnapshotWithoutTxs(Long.MAX_VALUE, Long.MAX_VALUE, MVCC_READ_OP_CNTR, MVCC_COUNTER_NA); /** */ private static final MvccClosure getVisibleState = new GetVisibleState(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java index cc634fae5daca..6fb200dcda924 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java @@ -911,12 +911,16 @@ GridCacheMvccCandidate anyOwner() { return false; } + /** {@inheritDoc} */ + @Nullable @Override public CacheObject mvccPeek(boolean onheapOnly) { + return null; + } + /** {@inheritDoc} */ @Nullable @Override public CacheObject peek(boolean heap, boolean offheap, AffinityTopologyVersion topVer, - @Nullable IgniteCacheExpiryPolicy plc) - { + @Nullable IgniteCacheExpiryPolicy plc) { return null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccOperationChecksTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccOperationChecksTest.java index 5aedf17089d86..d8b93aae2cdfb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccOperationChecksTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccOperationChecksTest.java @@ -103,14 +103,6 @@ public void testLockOperationsUnsupported() throws Exception { checkOperationUnsupported("lockAll", m("Lock"), t(Collection.class), Collections.singleton(1)); } - /** - * @throws Exception if failed. - */ - public void testPeekOperationsUnsupported() throws Exception { - checkOperationUnsupported("localPeek", m("Peek"), t(Object.class, CachePeekMode[].class), 1, - new CachePeekMode[]{CachePeekMode.NEAR}); - } - /** * @throws Exception if failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachePeekTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachePeekTest.java new file mode 100644 index 0000000000000..539283b0c0aee --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachePeekTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.mvcc; + +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.transactions.Transaction; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** */ +public class MvccCachePeekTest extends CacheMvccAbstractTest { + /** */ + private interface ThrowingRunnable { + /** */ + void run() throws Exception; + } + + /** */ + private IgniteCache cache; + + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return CacheMode.PARTITIONED; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + startGridsMultiThreaded(3); + } + + /** + * @throws Exception if failed. + */ + public void testPeek() throws Exception { + doWithCache(this::checkPeekSerial); + doWithCache(this::checkPeekDoesNotSeeAbortedVersions); + doWithCache(this::checkPeekDoesNotSeeActiveVersions); + doWithCache(this::checkPeekOnheap); + doWithCache(this::checkPeekNearCache); + } + + /** */ + private void doWithCache(ThrowingRunnable action) throws Exception { + cache = grid(0).getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setBackups(1) + .setCacheMode(cacheMode())); + + try { + action.run(); + } + finally { + cache.destroy(); + } + } + + /** */ + private void checkPeekSerial() throws Exception { + Stream.of(primaryKey(cache), backupKey(cache)).forEach(key -> { + assertNull(cache.localPeek(key)); + + cache.put(key, 1); + + assertEquals(1, cache.localPeek(key)); + + cache.put(key, 2); + + assertEquals(2, cache.localPeek(key)); + }); + } + + /** */ + private void checkPeekDoesNotSeeAbortedVersions() throws Exception { + Integer pk = primaryKey(cache); + + cache.put(pk, 1); + + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.put(pk, 2); + + tx.rollback(); + } + + assertEquals(1, cache.localPeek(pk)); + } + + /** */ + private void checkPeekDoesNotSeeActiveVersions() throws Exception { + Integer pk = primaryKey(cache); + + cache.put(pk, 1); + + CountDownLatch writeCompleted = new CountDownLatch(1); + CountDownLatch checkCompleted = new CountDownLatch(1); + + IgniteInternalFuture fut = GridTestUtils.runAsync(() -> { + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.put(pk, 2); + + writeCompleted.countDown(); + checkCompleted.await(); + + tx.commit(); + } + + return null; + }); + + writeCompleted.await(); + + assertEquals(1, cache.localPeek(pk)); + + checkCompleted.countDown(); + + fut.get(); + } + + /** */ + private void checkPeekOnheap() throws Exception { + Stream.of(primaryKey(cache), backupKey(cache), nearKey(cache)).forEach(key -> { + cache.put(key, 1); + + assertNull(cache.localPeek(key, CachePeekMode.ONHEAP)); + }); + } + + /** */ + private void checkPeekNearCache() throws Exception { + Stream.of(primaryKey(cache), backupKey(cache), nearKey(cache)).forEach(key -> { + cache.put(key, 1); + + assertNull(cache.localPeek(key, CachePeekMode.NEAR)); + }); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java index 014634417cd24..0c086f50ad284 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java @@ -18,6 +18,9 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.internal.processors.cache.IgniteCacheTxPeekModesTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheTxReplicatedPeekModesTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccClusterRestartTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccConfigurationValidationTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccIteratorWithConcurrentTransactionTest; @@ -32,8 +35,11 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSizeWithConcurrentTransactionTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTransactionsTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccVacuumTest; +import org.apache.ignite.internal.processors.cache.mvcc.MvccCachePeekTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + /** * */ @@ -55,6 +61,11 @@ public static TestSuite suite() { suite.addTestSuite(CacheMvccRemoteTxOnNearNodeStartTest.class); + suite.addTestSuite(MvccCachePeekTest.class); + + suite.addTestSuite(MvccIgniteCacheTxPeekModesTest.class); + suite.addTestSuite(MvccIgniteCacheTxReplicatedPeekModesTest.class); + // Concurrent ops tests. suite.addTestSuite(CacheMvccIteratorWithConcurrentTransactionTest.class); suite.addTestSuite(CacheMvccLocalEntriesWithConcurrentTransactionTest.class); @@ -69,4 +80,34 @@ public static TestSuite suite() { return suite; } + + /** */ + public static class MvccIgniteCacheTxPeekModesTest extends IgniteCacheTxPeekModesTest { + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL_SNAPSHOT; + } + + /** {@inheritDoc} */ + @Override public void testLocalEntries() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10167"); + + super.testLocalEntries(); + } + } + + /** */ + public static class MvccIgniteCacheTxReplicatedPeekModesTest extends IgniteCacheTxReplicatedPeekModesTest { + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL_SNAPSHOT; + } + + /** {@inheritDoc} */ + @Override public void testLocalEntries() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10167"); + + super.testLocalEntries(); + } + } } From 49fb3d38c6dfac790c98fd43ac4704c3e14cdbc2 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Tue, 20 Nov 2018 15:26:31 +0300 Subject: [PATCH 157/403] IGNITE-9828 MVCC: Notify Continuous Query manager about missed updates during PME. This closes #5189. (cherry picked from commit 74f312e0c863824f1cefdd589a7a0d7da140a276) --- .../processors/cache/GridCacheMapEntry.java | 12 +- .../GridCachePartitionExchangeManager.java | 4 +- .../cache/IgniteCacheOffheapManager.java | 4 +- .../cache/IgniteCacheOffheapManagerImpl.java | 4 +- .../cache/PartitionUpdateCounter.java | 28 +- .../GridDistributedTxRemoteAdapter.java | 2 + .../dht/GridDhtTransactionalCacheAdapter.java | 53 ++- .../dht/GridDhtTxAbstractEnlistFuture.java | 10 +- .../GridDhtPartitionsExchangeFuture.java | 49 +++ .../topology/GridClientPartitionTopology.java | 8 +- .../dht/topology/GridDhtLocalPartition.java | 7 +- .../topology/GridDhtPartitionTopology.java | 9 +- .../GridDhtPartitionTopologyImpl.java | 51 ++- .../cache/mvcc/MvccCachingManager.java | 30 +- .../persistence/GridCacheOffheapManager.java | 5 +- .../CacheContinuousQueryManager.java | 33 ++ .../cache/transactions/IgniteTxAdapter.java | 3 + .../cache/transactions/IgniteTxHandler.java | 2 +- .../continuous/GridContinuousProcessor.java | 6 +- ...nuousQueryAsyncFailoverMvccTxSelfTest.java | 53 +++ ...ntinuousQueryFailoverAbstractSelfTest.java | 90 ++++- ...QueryFailoverMvccTxReplicatedSelfTest.java | 31 ++ ...ContinuousQueryFailoverMvccTxSelfTest.java | 48 +++ .../CacheMvccBasicContinuousQueryTest.java | 363 ++++++++++++++++++ .../IgniteCacheQuerySelfTestSuite4.java | 6 + 25 files changed, 821 insertions(+), 90 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxReplicatedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index a68a4d317e2db..bd2d5afc3dfc7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -6679,12 +6679,14 @@ private IgniteBiTuple runEntryProcessor(CacheInvokeEntry runEntryProcessor(CacheInvokeEntry runEntryProcessor(CacheInvokeEntry cursor(int cacheId, KeyCacheObject low /** * Flushes pending update counters closing all possible gaps. + * + * @return Even-length array of pairs [start, end] for each gap. */ - public void finalizeUpdateCountres(); + GridLongList finalizeUpdateCounters(); /** * Preload a store into page memory. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 5c3398cafb806..8f6c51df91359 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -1568,8 +1568,8 @@ void decrementSize(int cacheId) { } /** {@inheritDoc} */ - @Override public void finalizeUpdateCountres() { - pCntr.finalizeUpdateCountres(); + @Override public GridLongList finalizeUpdateCounters() { + return pCntr.finalizeUpdateCounters(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java index fe4470875b7e6..39d8d5fcc1c21 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java @@ -20,6 +20,7 @@ import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.util.GridLongList; import org.jetbrains.annotations.NotNull; /** @@ -180,14 +181,31 @@ private void offer(Item item) { /** * Flushes pending update counters closing all possible gaps. + * + * @return Even-length array of pairs [start, end] for each gap. */ - public synchronized void finalizeUpdateCountres() { - Item last = queue.pollLast(); + public synchronized GridLongList finalizeUpdateCounters() { + Item item = poll(); + + GridLongList gaps = null; + + while (item != null) { + if (gaps == null) + gaps = new GridLongList((queue.size() + 1) * 2); + + long start = cntr.get() + 1; + long end = item.start; - if (last != null) - update(last.start + last.delta); + gaps.add(start); + gaps.add(end); + + // Close pending ranges. + update(item.start + item.delta); + + item = poll(); + } - queue.clear(); + return gaps; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java index 51e6e66e76f6e..0db9b4e79a4df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java @@ -931,6 +931,8 @@ public void forceCommit() throws IgniteCheckedException { catch (IgniteCheckedException | RuntimeException | Error e) { state(UNKNOWN); + U.error(log, "Error during tx rollback.", e); + if (e instanceof IgniteCheckedException) throw new IgniteException(e); else if (e instanceof RuntimeException) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index edd28b44a47bc..99c5111807447 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -1962,23 +1962,24 @@ private void processNearTxQueryResultsEnlistRequest(UUID nodeId, final GridNearT req.subjectId(), req.taskNameHash()); } - catch (IgniteCheckedException | IgniteException ex) { + catch (Throwable e) { GridNearTxQueryResultsEnlistResponse res = new GridNearTxQueryResultsEnlistResponse(req.cacheId(), req.futureId(), req.miniId(), req.version(), - ex); + e); try { ctx.io().send(nearNode, res, ctx.ioPolicy()); } - catch (IgniteCheckedException e) { - U.error(log, "Failed to send near enlist response [" + - "txId=" + req.version() + - ", node=" + nodeId + - ", res=" + res + ']', e); + catch (IgniteCheckedException ioEx) { + U.error(log, "Failed to send near enlist response " + + "[txId=" + req.version() + ", node=" + nodeId + ", res=" + res + ']', ioEx); } + if (e instanceof Error) + throw (Error) e; + return; } @@ -2227,26 +2228,6 @@ private void processNearTxQueryResultsEnlistResponse(UUID nodeId, final GridNear fut.onResult(nodeId, res); } - /** - * @param primary Primary node. - * @param req Request. - * @param e Error. - */ - private void onError(UUID primary, GridDhtTxQueryEnlistRequest req, Throwable e) { - GridDhtTxQueryEnlistResponse res = new GridDhtTxQueryEnlistResponse(ctx.cacheId(), - req.dhtFutureId(), - req.batchId(), - e); - - try { - ctx.io().send(primary, res, ctx.ioPolicy()); - } - catch (IgniteCheckedException ioEx) { - U.error(log, "Failed to send DHT enlist reply to primary node [node: " + primary + ", req=" + req + - ']', ioEx); - } - } - /** * @param primary Primary node. * @param req Message. @@ -2318,8 +2299,22 @@ private void processDhtTxQueryEnlistRequest(UUID primary, GridDhtTxQueryEnlistRe req + ']', ioEx); } } - catch (IgniteCheckedException e) { - onError(primary, req, e); + catch (Throwable e) { + GridDhtTxQueryEnlistResponse res = new GridDhtTxQueryEnlistResponse(ctx.cacheId(), + req.dhtFutureId(), + req.batchId(), + e); + + try { + ctx.io().send(primary, res, ctx.ioPolicy()); + } + catch (IgniteCheckedException ioEx) { + U.error(log, "Failed to send DHT enlist reply to primary node " + + "[node: " + primary + ", req=" + req + ']', ioEx); + } + + if (e instanceof Error) + throw (Error) e; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java index 68669b7460f22..641931de273db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java @@ -644,7 +644,7 @@ private void processEntry(GridDhtCacheEntry entry, EnlistOperation op, updRes.oldValue(), tx.local(), tx.topologyVersion(), mvccSnapshot, cctx.cacheId(), tx, null, -1); if (op != EnlistOperation.LOCK) - addToBatch(entry.key(), val, updRes.mvccHistory(), entry.context().cacheId()); + addToBatch(entry.key(), val, updRes.mvccHistory(), entry.context().cacheId(), op); } /** @@ -657,7 +657,7 @@ private void processEntry(GridDhtCacheEntry entry, EnlistOperation op, * @param cacheId Cache Id. */ private void addToBatch(KeyCacheObject key, Message val, List hist, - int cacheId) throws IgniteCheckedException { + int cacheId, EnlistOperation op) throws IgniteCheckedException { List backups = backupNodes(key); int part = cctx.affinity().partition(key); @@ -695,13 +695,15 @@ private void addToBatch(KeyCacheObject key, Message val, List nonLocalCacheGroupDescriptors() { .collect(Collectors.toList()); } + /** + * Collects non local cache groups. + * + * @return Collection of non local cache groups. + */ + private List nonLocalCacheGroups() { + return cctx.cache().cacheGroups().stream() + .filter(grp -> !grp.isLocal() && !cacheGroupStopping(grp.groupId())) + .collect(Collectors.toList()); + } + /** * Validates that partition update counters and cache sizes for all caches are consistent. */ @@ -3541,6 +3555,41 @@ private void assignPartitionsStates() { log.info("Partitions assignment performed in " + (System.currentTimeMillis() - time) + " ms."); } + /** + * Removes gaps in the local update counters. Gaps in update counters are possible on backup node when primary + * failed to send update counter deltas to backup. + */ + private void finalizePartitionCounters() { + long time = System.currentTimeMillis(); + + try { + int parallelismLvl = cctx.kernalContext().config().getSystemThreadPoolSize(); + + // Reserve at least 2 threads for system operations. + parallelismLvl = Math.max(1, parallelismLvl - 2); + + if (parallelismLvl > 1) { + U.doInParallel(parallelismLvl, + cctx.kernalContext().getSystemExecutorService(), + nonLocalCacheGroups(), + grp -> { + grp.topology().finalizeUpdateCounters(); + + return null; + } + ); + } + else + nonLocalCacheGroups().forEach(grp -> grp.topology().finalizeUpdateCounters()); + } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to finalize partition counters", e); + } + + if (log.isInfoEnabled()) + log.info("Partition counters finalization performed in " + (System.currentTimeMillis() - time) + " ms."); + } + /** * @param finishState State. * @param msg Request. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java index cd6e254efcf50..3c4eba15337d7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java @@ -1234,11 +1234,15 @@ private void removeNode(UUID nodeId) { } /** {@inheritDoc} */ - @Override public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros, - boolean finalizeCntrsBeforeCollecting) { + @Override public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros) { return CachePartitionPartialCountersMap.EMPTY; } + /** {@inheritDoc} */ + @Override public void finalizeUpdateCounters() { + // No-op. + } + /** {@inheritDoc} */ @Override public Map partitionSizes() { return Collections.emptyMap(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 588c8b4068f18..2557bb3946781 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner; +import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -1364,9 +1365,11 @@ private static long setSize(long state, int size) { /** * Flushes pending update counters closing all possible gaps. + * + * @return Even-length array of pairs [start, end] for each gap. */ - public void finalizeUpdateCountres() { - store.finalizeUpdateCountres(); + public GridLongList finalizeUpdateCounters() { + return store.finalizeUpdateCounters(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index bdf16da378144..2f5e57bace9dc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -345,6 +345,11 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, */ public Collection lostPartitions(); + /** + * Pre-processes partition update counters before exchange. + */ + void finalizeUpdateCounters(); + /** * @return Partition update counters. */ @@ -352,11 +357,9 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, /** * @param skipZeros {@code True} for adding zero counter to map. - * @param finalizeCntrsBeforeCollecting {@code True} indicates that partition counters should be finalized. * @return Partition update counters. */ - public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros, - boolean finalizeCntrsBeforeCollecting); + public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros); /** * @return Partition cache sizes. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index aeb71e9243e27..6c08ecab9a50b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -46,6 +46,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents; +import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; @@ -58,6 +59,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.util.F0; import org.apache.ignite.internal.util.GridAtomicLong; +import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.GridPartitionStateMap; import org.apache.ignite.internal.util.StripedCompositeReadWriteLock; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -2628,6 +2630,49 @@ private void removeNode(UUID nodeId) { } } + /** + * Pre-processes partition update counters before exchange. + */ + @Override public void finalizeUpdateCounters() { + if (!grp.mvccEnabled()) + return; + + // It is need to acquire checkpoint lock before topology lock acquiring. + ctx.database().checkpointReadLock(); + + try { + lock.readLock().lock(); + + try { + for (int i = 0; i < locParts.length(); i++) { + GridDhtLocalPartition part = locParts.get(i); + + if (part != null && part.state().active()) { + // We need to close all gaps in partition update counters sequence. We assume this finalizing is + // happened on exchange and hence all txs are completed. Therefore each gap in update counters + // sequence is a result of undelivered DhtTxFinishMessage on backup (sequences on primary nodes + // do not have gaps). Here we close these gaps and asynchronously notify continuous query engine + // about the skipped events. + AffinityTopologyVersion topVer = ctx.exchange().readyAffinityVersion(); + + GridLongList gaps = part.finalizeUpdateCounters(); + + if (gaps != null) { + for (GridCacheContext ctx0 : grp.caches()) + ctx0.continuousQueries().closeBackupUpdateCountersGaps(ctx0, part.id(), topVer, gaps); + } + } + } + } + finally { + lock.readLock().unlock(); + } + } + finally { + ctx.database().checkpointReadUnlock(); + } + } + /** {@inheritDoc} */ @Override public CachePartitionFullCountersMap fullUpdateCounters() { lock.readLock().lock(); @@ -2641,8 +2686,7 @@ private void removeNode(UUID nodeId) { } /** {@inheritDoc} */ - @Override public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros, - boolean finalizeCntrsBeforeCollecting) { + @Override public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros) { lock.readLock().lock(); try { @@ -2663,9 +2707,6 @@ private void removeNode(UUID nodeId) { if (part == null) continue; - if (finalizeCntrsBeforeCollecting) - part.finalizeUpdateCountres(); - long updCntr = part.updateCounter(); long initCntr = part.initialUpdateCounter(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachingManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachingManager.java index 259f69b3c706c..8f83b6e5b3e1d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachingManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCachingManager.java @@ -40,8 +40,10 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.TxCounters; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; @@ -146,16 +148,23 @@ public void onTxFinished(IgniteInternalTx tx, boolean commit) throws IgniteCheck Map cached = buf.getCached(); - if (F.isEmpty(cached) || !commit) + if (F.isEmpty(cached)) return; TxCounters txCntrs = tx.txCounters(false); - assert txCntrs != null; + assert txCntrs != null || !commit; - Collection cntrsColl = txCntrs.updateCounters(); + if (txCntrs == null) + return; + + Collection cntrsColl = txCntrs.updateCounters(); - assert !F.isEmpty(cntrsColl) : cntrsColl; + if (F.isEmpty(cntrsColl)) { + assert !commit; + + return; + } // cacheId -> partId -> initCntr -> cntr + delta. Map>> cntrsMap = new HashMap<>(); @@ -219,8 +228,8 @@ public void onTxFinished(IgniteInternalTx tx, boolean commit) throws IgniteCheck contQryMgr.onEntryUpdated( lsnrCol, e.key(), - e.value(), - e.oldValue(), + commit ? e.value() : null, // Force skip update counter if rolled back. + commit ? e.oldValue() : null, // Force skip update counter if rolled back. false, e.key().partition(), tx.local(), @@ -253,7 +262,7 @@ public boolean needDrReplicate(GridCacheContext ctx0, KeyCacheObject key) { * @return Map of listeners to be notified by this update. */ public Map continuousQueryListeners(GridCacheContext ctx0, @Nullable IgniteInternalTx tx, KeyCacheObject key) { - boolean internal = key.internal() || !ctx0.userCache(); + boolean internal = key != null && key.internal() || !ctx0.userCache(); return ctx0.continuousQueries().notifyContinuousQueries(tx) ? ctx0.continuousQueries().updateListeners(internal, false) : null; @@ -268,9 +277,11 @@ private static class EnlistBuffer { private IgniteUuid lastFutId; /** Main buffer for entries. */ + @GridToStringInclude private Map cached = new LinkedHashMap<>(); /** Pending entries. */ + @GridToStringInclude private SortedMap> pending; /** @@ -337,5 +348,10 @@ private void flushPending() { pending.clear(); } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(EnlistBuffer.class, this); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 7e33de11ba05c..34b534e822114 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -1862,12 +1862,11 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void finalizeUpdateCountres() { + @Override public GridLongList finalizeUpdateCounters() { try { CacheDataStore delegate0 = init0(true); - if (delegate0 != null) - delegate0.finalizeUpdateCountres(); + return delegate0 != null ? delegate0.finalizeUpdateCounters() : null; } catch (IgniteCheckedException e) { throw new IgniteException(e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java index 6bd3fc2cc549f..4c399bf8cb3f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java @@ -58,6 +58,7 @@ import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.GridCacheManagerAdapter; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; @@ -67,6 +68,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.continuous.GridContinuousHandler; import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; +import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.StripedCompositeReadWriteLock; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CI2; @@ -274,6 +276,37 @@ private void skipUpdateEvent(Map lsnrs, return skipCtx; } + /** + * For cache updates in shared cache group need notify others caches CQ listeners + * that generated counter should be skipped. + * + * @param cctx Cache context. + * @param part Partition. + * @param topVer Topology version. + * @param gaps Even-length array of pairs [start, end] for each gap. + */ + @Nullable public void closeBackupUpdateCountersGaps(GridCacheContext cctx, + int part, + AffinityTopologyVersion topVer, + GridLongList gaps) { + assert gaps != null && gaps.size() % 2 == 0; + + for (int i = 0; i < gaps.size() / 2; i++) { + long gapStart = gaps.get(i * 2); + long gapStop = gaps.get(i * 2 + 1); + + /* + * No user listeners should be called by this invocation. In the common case of partitioned cache or + * replicated cache with non-local-only listener gaps (dummy filtered CQ events) will be added to the + * backup queue without passing it to any listener. In the special case of local-only listener on + * replicated cache there is no backup queues used at all and therefore no gaps occur - all unfiltered + * events are passed to listeners upon arrive. + */ + for (long cntr = gapStart; cntr <= gapStop; cntr++) + skipUpdateEvent(lsnrs, null, part, cntr, false, topVer); + } + } + /** * @param internal Internal entry flag (internal key or not user cache). * @param preload Whether update happened during preloading. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index 399359bf11ab7..19ec92416b48b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -275,6 +275,7 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement protected ConsistentIdMapper consistentIdMapper; /** Mvcc tx update snapshot. */ + @GridToStringInclude protected volatile MvccSnapshot mvccSnapshot; /** Rollback finish future. */ @@ -282,6 +283,8 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement private volatile IgniteInternalFuture rollbackFut; /** */ + @SuppressWarnings("unused") + @GridToStringExclude private volatile TxCounters txCounters; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 0c51e1502aae9..3d6886eacd5c7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -1891,7 +1891,7 @@ public void mvccEnlistBatch(GridDhtTxRemote tx, GridCacheContext ctx, EnlistOper invokeArgs = invokeVal.invokeArgs(); } - assert entryProc != null || !op.isInvoke(); + assert entryProc != null || !op.isInvoke() : "entryProc=" + entryProc + ", op=" + op; GridDhtCacheEntry entry = dht.entryExx(key, tx.topologyVersion()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java index 88bfbf4e15a50..f95740efdd077 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java @@ -1423,7 +1423,7 @@ private void processStartRequest(ClusterNode node, StartRoutineDiscoveryMessage if (cache != null && !cache.isLocal() && cache.context().userCache()) req.addUpdateCounters(ctx.localNodeId(), - toCountersMap(cache.context().topology().localUpdateCounters(false, false))); + toCountersMap(cache.context().topology().localUpdateCounters(false))); } } @@ -1564,7 +1564,7 @@ private void processStartRequestV2(final AffinityTopologyVersion topVer, if (cache != null && !cache.isLocal() && cache.context().userCache()) { CachePartitionPartialCountersMap cntrsMap = - cache.context().topology().localUpdateCounters(false, false); + cache.context().topology().localUpdateCounters(false); cntrs = U.marshal(marsh, cntrsMap); } @@ -2504,7 +2504,7 @@ private void onAllRemoteRegistered( if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode()) cntrsPerNode.put(ctx.localNodeId(), - toCountersMap(cctx.topology().localUpdateCounters(false, false))); + toCountersMap(cctx.topology().localUpdateCounters(false))); routine.handler().updateCounters(topVer, cntrsPerNode, cntrs); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java new file mode 100644 index 0000000000000..334d2190c998b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.query.continuous; + +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; + +/** + * + */ +public class CacheContinuousQueryAsyncFailoverMvccTxSelfTest extends CacheContinuousQueryFailoverAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return PARTITIONED; + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL_SNAPSHOT; + } + + /** {@inheritDoc} */ + @Override protected boolean asyncCallback() { + return true; + } + + /** {@inheritDoc} */ + @Override public void testBackupQueueEvict() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + } + + /** {@inheritDoc} */ + @Override public void testLeftPrimaryAndBackupNodes() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10047"); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java index 0fef7b21b183f..e7c22611f31a8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java @@ -74,8 +74,8 @@ import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.continuous.GridContinuousHandler; import org.apache.ignite.internal.processors.continuous.GridContinuousMessage; import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor; @@ -87,6 +87,7 @@ import org.apache.ignite.internal.util.typedef.PAX; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.T3; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteAsyncCallback; @@ -104,6 +105,7 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionRollbackException; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; @@ -111,6 +113,8 @@ import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; /** * @@ -521,7 +525,7 @@ private void checkPartCounter(int nodes, int killedNodeIdx, Map u Affinity aff = grid(i).affinity(DEFAULT_CACHE_NAME); CachePartitionPartialCountersMap act = grid(i).cachex(DEFAULT_CACHE_NAME).context().topology() - .localUpdateCounters(false, false); + .localUpdateCounters(false); for (Map.Entry e : updCntrs.entrySet()) { if (aff.mapPartitionToPrimaryAndBackups(e.getKey()).contains(grid(i).localNode())) { @@ -760,6 +764,8 @@ public void testLeftPrimaryAndBackupNodes() throws Exception { } }, 5000L); + awaitPartitionMapExchange(); + for (; keyIter < keys.size(); keyIter++) { int key = keys.get(keyIter); @@ -784,7 +790,18 @@ public void testLeftPrimaryAndBackupNodes() throws Exception { expEvts.add(new T3<>((Object)key, (Object)val, (Object)key)); } - clnCache.put(key, val); + boolean updated = false; + + while (!updated) { + try { + clnCache.put(key, val); + + updated = true; + } + catch (Exception ignore) { + assertEquals(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, atomicityMode()); + } + } filtered = !filtered; } @@ -977,8 +994,8 @@ private void checkBackupQueue(int backups, boolean updateFromClient) throws Exce T2 t = updates.get(key); if (updateFromClient) { - if (atomicityMode() == CacheAtomicityMode.TRANSACTIONAL) { - try (Transaction tx = qryClient.transactions().txStart()) { + if (atomicityMode() != CacheAtomicityMode.ATOMIC) { + try (Transaction tx = qryClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { qryClientCache.put(key, key); tx.commit(); @@ -993,8 +1010,8 @@ private void checkBackupQueue(int backups, boolean updateFromClient) throws Exce qryClientCache.put(key, key); } else { - if (atomicityMode() == CacheAtomicityMode.TRANSACTIONAL) { - try (Transaction tx = ignite.transactions().txStart()) { + if (atomicityMode() != CacheAtomicityMode.ATOMIC) { + try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { cache.put(key, key); tx.commit(); @@ -1755,18 +1772,30 @@ private void failoverStartStopFilter(int backups) throws Exception { if (filtered) val = -val; - if (processorPut && prevVal != null) { - qryClnCache.invoke(key, new CacheEntryProcessor() { - @Override public Void process(MutableEntry entry, - Object... arguments) throws EntryProcessorException { - entry.setValue(arguments[0]); + boolean updated = false; + + while (!updated) { + try { + if (processorPut && prevVal != null) { + qryClnCache.invoke(key, new CacheEntryProcessor() { + @Override public Void process(MutableEntry entry, + Object... arguments) throws EntryProcessorException { + entry.setValue(arguments[0]); - return null; + return null; + } + }, val); } - }, val); + else + qryClnCache.put(key, val); + + updated = true; + } + catch (CacheException e) { + assertTrue(X.hasCause(e, TransactionRollbackException.class)); + assertSame(atomicityMode(), CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } } - else - qryClnCache.put(key, val); processorPut = !processorPut; @@ -2020,7 +2049,20 @@ public void testMultiThreadedFailover() throws Exception { Integer val = valCntr.incrementAndGet(); - Integer prevVal = (Integer)qryClnCache.getAndPut(key, val); + Integer prevVal = null; + + boolean updated = false; + + while (!updated) { + try { + prevVal = (Integer)qryClnCache.getAndPut(key, val); + + updated = true; + } + catch (CacheException e) { + assertSame(atomicityMode(), CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } + } expEvts.get(threadId).add(new T3<>((Object)key, (Object)val, (Object)prevVal)); @@ -2114,7 +2156,19 @@ public void testMultiThreaded() throws Exception { @Override public Object call() throws Exception { Integer val0 = val.getAndIncrement(); - cache.put(key, val0); + boolean updated = false; + + while (!updated) { + try { + cache.put(key, val0); + + updated = true; + } + catch (CacheException e) { + assertTrue(X.hasCause(e, TransactionRollbackException.class)); + assertSame(atomicityMode(), CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } + } return null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxReplicatedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxReplicatedSelfTest.java new file mode 100644 index 0000000000000..2576d23f6839a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxReplicatedSelfTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.query.continuous; + +import org.apache.ignite.cache.CacheMode; + +import static org.apache.ignite.cache.CacheMode.REPLICATED; + +/** + * + */ +public class CacheContinuousQueryFailoverMvccTxReplicatedSelfTest extends CacheContinuousQueryFailoverMvccTxSelfTest { + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return REPLICATED; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java new file mode 100644 index 0000000000000..76d1689715562 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.query.continuous; + +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; + +/** + * + */ +public class CacheContinuousQueryFailoverMvccTxSelfTest extends CacheContinuousQueryFailoverAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return PARTITIONED; + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL_SNAPSHOT; + } + + /** {@inheritDoc} */ + @Override public void testBackupQueueEvict() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + } + + /** {@inheritDoc} */ + @Override public void testLeftPrimaryAndBackupNodes() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10047"); + } +} \ No newline at end of file diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBasicContinuousQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBasicContinuousQueryTest.java index ed97b1bec710d..e08341a24a033 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBasicContinuousQueryTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBasicContinuousQueryTest.java @@ -17,11 +17,14 @@ package org.apache.ignite.internal.processors.cache.mvcc; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; import javax.cache.Cache; import javax.cache.CacheException; import javax.cache.event.CacheEntryEvent; @@ -29,23 +32,35 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.Affinity; import org.apache.ignite.cache.query.ContinuousQuery; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareRequest; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager; +import org.apache.ignite.internal.processors.continuous.GridContinuousMessage; import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.transactions.Transaction; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.processors.cache.mvcc.MvccCachingManager.TX_SIZE_THRESHOLD; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; +import static org.apache.ignite.transactions.TransactionState.PREPARING; +import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK; /** * Basic continuous queries test with enabled mvcc. @@ -227,4 +242,352 @@ public void testCachingMaxSize() throws Exception { } }, CacheException.class, "Failed to run update. Transaction is too large. Consider reducing transaction size"); } + + /** + * @throws Exception If failed. + */ + public void testUpdateCountersGapClosedSimplePartitioned() throws Exception { + checkUpdateCountersGapIsProcessedSimple(CacheMode.PARTITIONED); + } + + /** + * @throws Exception If failed. + */ + public void testUpdateCountersGapClosedSimpleReplicated() throws Exception { + checkUpdateCountersGapIsProcessedSimple(CacheMode.REPLICATED); + } + + /** + * @throws Exception if failed. + */ + private void checkUpdateCountersGapIsProcessedSimple(CacheMode cacheMode) throws Exception { + testSpi = true; + + int srvCnt = 4; + + startGridsMultiThreaded(srvCnt); + + client = true; + + IgniteEx nearNode = startGrid(srvCnt); + + IgniteCache cache = nearNode.createCache( + cacheConfiguration(cacheMode, FULL_SYNC, srvCnt - 1, srvCnt) + .setIndexedTypes(Integer.class, Integer.class)); + + IgniteEx primary = grid(0); + + List keys = primaryKeys(primary.cache(DEFAULT_CACHE_NAME), 3); + + ContinuousQuery qry = new ContinuousQuery<>(); + + List arrivedEvts = new ArrayList<>(); + + CountDownLatch latch = new CountDownLatch(2); + + qry.setLocalListener(new CacheEntryUpdatedListener() { + @Override public void onUpdated(Iterable> evts) { + for (CacheEntryEvent e : evts) { + arrivedEvts.add(e); + + latch.countDown(); + } + } + }); + + QueryCursor> cur = nearNode.cache(DEFAULT_CACHE_NAME).query(qry); + + // Initial value. + cache.query(new SqlFieldsQuery("insert into Integer(_key, _val) values(?, 42)").setArgs(keys.get(0))).getAll(); + + Transaction txA = nearNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ); + + // prevent first transaction prepare on backups + TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(primary); + + spi.blockMessages(new IgniteBiPredicate() { + private final AtomicInteger limiter = new AtomicInteger(); + + @Override public boolean apply(ClusterNode node, Message msg) { + if (msg instanceof GridDhtTxPrepareRequest) + return limiter.getAndIncrement() < srvCnt - 1; + + if (msg instanceof GridContinuousMessage) + return true; + + return false; + } + }); + + cache.query(new SqlFieldsQuery("insert into Integer(_key, _val) values(?, 42)").setArgs(keys.get(1))).getAll(); + + txA.commitAsync(); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return nearNode.context().cache().context().tm().activeTransactions().stream().allMatch(tx -> tx.state() == PREPARING); + } + }, 3_000); + + GridTestUtils.runAsync(() -> { + try (Transaction txB = nearNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.query(new SqlFieldsQuery("insert into Integer(_key, _val) values(?, 42)").setArgs(keys.get(2))); + + txB.commit(); + } + }).get(); + + long primaryUpdCntr = getUpdateCounter(primary, keys.get(0)); + + assertEquals(3, primaryUpdCntr); // There were three updates. + + // drop primary + stopGrid(primary.name()); + + // Wait all txs are rolled back. + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + boolean allRolledBack = true; + + for (int i = 1; i < srvCnt; i++) { + boolean rolledBack = grid(i).context().cache().context().tm().activeTransactions().stream().allMatch(tx -> tx.state() == ROLLED_BACK); + + allRolledBack &= rolledBack; + } + + return allRolledBack; + } + }, 3_000); + + for (int i = 1; i < srvCnt; i++) { + IgniteCache backupCache = grid(i).cache(DEFAULT_CACHE_NAME); + + int size = backupCache.query(new SqlFieldsQuery("select * from Integer")).getAll().size(); + + long backupCntr = getUpdateCounter(grid(i), keys.get(0)); + + assertEquals(2, size); + assertEquals(primaryUpdCntr, backupCntr); + } + + assertTrue(latch.await(3, SECONDS)); + + assertEquals(2, arrivedEvts.size()); + assertEquals(keys.get(0), arrivedEvts.get(0).getKey()); + assertEquals(keys.get(2), arrivedEvts.get(1).getKey()); + + cur.close(); + nearNode.close(); + } + + /** + * @throws Exception If failed. + */ + public void testUpdateCountersGapClosedPartitioned() throws Exception { + checkUpdateCountersGapsClosed(CacheMode.PARTITIONED); + } + + /** + * @throws Exception If failed. + */ + public void testUpdateCountersGapClosedReplicated() throws Exception { + checkUpdateCountersGapsClosed(CacheMode.REPLICATED); + } + + /** + * @throws Exception If failed. + */ + private void checkUpdateCountersGapsClosed(CacheMode cacheMode) throws Exception { + testSpi = true; + + int srvCnt = 4; + + startGridsMultiThreaded(srvCnt); + + IgniteEx nearNode = grid(srvCnt - 1); + + IgniteCache cache = nearNode.createCache( + cacheConfiguration(cacheMode, FULL_SYNC, srvCnt - 1, srvCnt) + .setIndexedTypes(Integer.class, Integer.class)); + + IgniteEx primary = grid(0); + + Affinity aff = nearNode.affinity(cache.getName()); + + int[] nearBackupParts = aff.backupPartitions(nearNode.localNode()); + + int[] primaryParts = aff.primaryPartitions(primary.localNode()); + + Collection nearSet = new HashSet<>(); + + for (int part : nearBackupParts) + nearSet.add(part); + + Collection primarySet = new HashSet<>(); + + for (int part : primaryParts) + primarySet.add(part); + + // We need backup partitions on the near node. + nearSet.retainAll(primarySet); + + List keys = singlePartKeys(primary.cache(DEFAULT_CACHE_NAME), 20, nearSet.iterator().next()); + + int range = 3; + + ContinuousQuery qry = new ContinuousQuery<>(); + + List arrivedEvts = new ArrayList<>(); + + CountDownLatch latch = new CountDownLatch(range * 2); + + qry.setLocalListener(new CacheEntryUpdatedListener() { + @Override public void onUpdated(Iterable> evts) { + for (CacheEntryEvent e : evts) { + arrivedEvts.add(e); + + latch.countDown(); + } + } + }); + + QueryCursor> cur = nearNode.cache(DEFAULT_CACHE_NAME).query(qry); + + // prevent first transaction prepare on backups + TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(primary); + + spi.blockMessages(new IgniteBiPredicate() { + private final AtomicInteger limiter = new AtomicInteger(); + + @Override public boolean apply(ClusterNode node, Message msg) { + if (msg instanceof GridDhtTxPrepareRequest) + return limiter.getAndIncrement() < srvCnt - 1; + + return false; + } + }); + + Transaction txA = primary.transactions().txStart(PESSIMISTIC, REPEATABLE_READ); + + for (int i = 0; i < range; i++) + primary.cache(DEFAULT_CACHE_NAME).put(keys.get(i), 2); + + txA.commitAsync(); + + GridTestUtils.runAsync(() -> { + try (Transaction tx = primary.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + for (int i = range; i < range * 2; i++) + primary.cache(DEFAULT_CACHE_NAME).put(keys.get(i), 1); + + tx.commit(); + } + }).get(); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return primary.context().cache().context().tm().activeTransactions().stream().allMatch(tx -> tx.state() == PREPARING); + } + }, 3_000); + + GridTestUtils.runAsync(() -> { + try (Transaction txB = primary.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + for (int i = range * 2; i < range * 3; i++) + primary.cache(DEFAULT_CACHE_NAME).put(keys.get(i), 3); + + txB.commit(); + } + }).get(); + + long primaryUpdCntr = getUpdateCounter(primary, keys.get(0)); + + assertEquals(range * 3, primaryUpdCntr); + + // drop primary + stopGrid(primary.name()); + + // Wait all txs are rolled back. + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + boolean allRolledBack = true; + + for (int i = 1; i < srvCnt; i++) { + boolean rolledBack = grid(i).context().cache().context().tm().activeTransactions().stream().allMatch(tx -> tx.state() == ROLLED_BACK); + + allRolledBack &= rolledBack; + } + + return allRolledBack; + } + }, 3_000); + + for (int i = 1; i < srvCnt; i++) { + IgniteCache backupCache = grid(i).cache(DEFAULT_CACHE_NAME); + + int size = backupCache.query(new SqlFieldsQuery("select * from Integer")).getAll().size(); + + long backupCntr = getUpdateCounter(grid(i), keys.get(0)); + + assertEquals(range * 2, size); + assertEquals(primaryUpdCntr, backupCntr); + } + + assertTrue(latch.await(5, SECONDS)); + + assertEquals(range * 2, arrivedEvts.size()); + + cur.close(); + nearNode.close(); + } + + /** + * @param primaryCache Cache. + * @param size Number of keys. + * @return Keys belong to a given part. + * @throws Exception If failed. + */ + private List singlePartKeys(IgniteCache primaryCache, int size, int part) throws Exception { + Ignite ignite = primaryCache.unwrap(Ignite.class); + + List res = new ArrayList<>(); + + final Affinity aff = ignite.affinity(primaryCache.getName()); + + final ClusterNode node = ignite.cluster().localNode(); + + assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return aff.primaryPartitions(node).length > 0; + } + }, 5000)); + + int cnt = 0; + + for (int key = 0; key < aff.partitions() * size * 10; key++) { + if (aff.partition(key) == part) { + res.add(key); + + if (++cnt == size) + break; + } + } + + assertEquals(size, res.size()); + + return res; + } + + /** + * @param node Node. + * @param key Key. + * @return Extracts update counter of partition which key belongs to. + */ + private long getUpdateCounter(IgniteEx node, Integer key) { + int partId = node.cachex(DEFAULT_CACHE_NAME).context().affinity().partition(key); + + GridDhtLocalPartition part = node.cachex(DEFAULT_CACHE_NAME).context().dht().topology().localPartition(partId); + + assert part != null; + + return part.updateCounter(); + } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java index 2aa3419764427..147eb47875259 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite4.java @@ -19,10 +19,13 @@ import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverMvccTxSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverTxReplicatedSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryAsyncFailoverTxSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverAtomicReplicatedSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverMvccTxReplicatedSelfTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverMvccTxSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverTxReplicatedSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverTxSelfTest; @@ -42,10 +45,13 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(CacheContinuousQueryFailoverAtomicReplicatedSelfTest.class); suite.addTestSuite(CacheContinuousQueryFailoverTxSelfTest.class); suite.addTestSuite(CacheContinuousQueryFailoverTxReplicatedSelfTest.class); + suite.addTestSuite(CacheContinuousQueryFailoverMvccTxSelfTest.class); + suite.addTestSuite(CacheContinuousQueryFailoverMvccTxReplicatedSelfTest.class); suite.addTestSuite(CacheContinuousQueryAsyncFailoverAtomicSelfTest.class); suite.addTestSuite(CacheContinuousQueryAsyncFailoverTxReplicatedSelfTest.class); suite.addTestSuite(CacheContinuousQueryAsyncFailoverTxSelfTest.class); + suite.addTestSuite(CacheContinuousQueryAsyncFailoverMvccTxSelfTest.class); return suite; } From 31ac50b41708bd4a89442a0764e515bc0488007e Mon Sep 17 00:00:00 2001 From: ipavlukhin Date: Wed, 21 Nov 2018 11:43:16 +0300 Subject: [PATCH 158/403] IGNITE-9622: MVCC: Prohibit non PESSIMISTIC REPEATABLE_READ transactions on MVCC-enabled caches. This closes #5074. (cherry picked from commit 4074a9b9b8cce04528337e0694966c09a9cdc049) --- .../processors/cache/GridCacheAdapter.java | 20 +- .../dht/colocated/GridDhtColocatedCache.java | 6 +- .../distributed/near/GridNearTxLocal.java | 30 +- .../processors/cache/mvcc/MvccUtils.java | 48 ++- .../mvcc/MvccUnsupportedTxModesTest.java | 316 ++++++++++++++++++ .../testsuites/IgniteCacheMvccTestSuite.java | 3 + 6 files changed, 382 insertions(+), 41 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUnsupportedTxModesTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index bd0d5835c289b..b16f389a5f125 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -147,6 +147,7 @@ import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionException; import org.apache.ignite.transactions.TransactionIsolation; import org.jetbrains.annotations.Nullable; @@ -1974,7 +1975,7 @@ protected final IgniteInternalFuture> getAllAsync0( return new GridFinishedFuture<>(e); } - tx = ctx.tm().threadLocalTx(ctx); + tx = checkCurrentTx(); } if (tx == null || tx.implicit()) { @@ -2315,6 +2316,19 @@ else if (storeEnabled) } } + /** */ + protected GridNearTxLocal checkCurrentTx() { + if (!ctx.mvccEnabled()) + return ctx.tm().threadLocalTx(ctx); + + try { + return MvccUtils.currentTx(ctx.kernalContext(), null); + } + catch (MvccUtils.UnsupportedTxModeException | MvccUtils.NonMvccTransactionException e) { + throw new TransactionException(e.getMessage()); + } + } + /** * @param topVer Affinity topology version for which load was performed. * @param loadKeys Keys to load. @@ -4240,7 +4254,7 @@ public void awaitLastFut() { awaitLastFut(); - GridNearTxLocal tx = ctx.tm().threadLocalTx(ctx); + GridNearTxLocal tx = checkCurrentTx(); if (tx == null || tx.implicit()) { TransactionConfiguration tCfg = CU.transactionConfiguration(ctx, ctx.kernalContext().config()); @@ -4343,7 +4357,7 @@ private IgniteInternalFuture asyncOp(final AsyncOp op) { if (log.isDebugEnabled()) log.debug("Performing async op: " + op); - GridNearTxLocal tx = ctx.tm().threadLocalTx(ctx); + GridNearTxLocal tx = checkCurrentTx(); CacheOperationContext opCtx = ctx.operationContextPerCall(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java index bbfb502c37521..6290a25570ee5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java @@ -189,7 +189,7 @@ public GridDistributedCacheEntry entryExx( if (keyCheck) validateCacheKey(key); - GridNearTxLocal tx = ctx.mvccEnabled() ? MvccUtils.tx(ctx.kernalContext()) : ctx.tm().threadLocalTx(ctx); + GridNearTxLocal tx = checkCurrentTx(); final CacheOperationContext opCtx = ctx.operationContextPerCall(); @@ -305,7 +305,7 @@ public GridDistributedCacheEntry entryExx( if (keyCheck) validateCacheKeys(keys); - GridNearTxLocal tx = (ctx.mvccEnabled()) ? MvccUtils.tx(ctx.kernalContext()) : ctx.tm().threadLocalTx(ctx); + GridNearTxLocal tx = checkCurrentTx(); final CacheOperationContext opCtx = ctx.operationContextPerCall(); @@ -345,7 +345,7 @@ public GridDistributedCacheEntry entryExx( assert mvccSnapshot != null; } catch (IgniteCheckedException ex) { - return new GridFinishedFuture(ex); + return new GridFinishedFuture<>(ex); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java index c7b2ad3a9e9b9..b698ba13c109e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java @@ -703,19 +703,6 @@ private IgniteInternalFuture putAsync0( } } - /** - * Validate Tx mode. - * - * @param ctx Cache context. - * @throws IgniteCheckedException If tx mode is not supported. - */ - protected void validateTxMode(GridCacheContext ctx) throws IgniteCheckedException { - if(!ctx.mvccEnabled() || pessimistic() && repeatableRead()) - return; - - throw new IgniteCheckedException("Only pessimistic repeatable read transactions are supported at the moment."); - } - /** * Internal method for put and transform operations in Mvcc mode. * Note: Only one of {@code map}, {@code transformMap} maps must be non-null. @@ -737,8 +724,6 @@ private IgniteInternalFuture mvccPutAllAsync0( @Nullable final CacheEntryPredicate filter ) { try { - validateTxMode(cacheCtx); - MvccUtils.requestSnapshot(cacheCtx, this); beforePut(cacheCtx, retval, true); @@ -1921,13 +1906,7 @@ private IgniteInternalFuture mvccRemoveAllAsync0( @Nullable final CacheEntryPredicate filter ) { try { - validateTxMode(cacheCtx); - - if (mvccSnapshot == null) { - MvccUtils.mvccTracker(cacheCtx, this); - - assert mvccSnapshot != null; - } + MvccUtils.requestSnapshot(cacheCtx, this); beforeRemove(cacheCtx, retval, true); } @@ -2188,13 +2167,6 @@ public IgniteInternalFuture> getAllAsync( if (F.isEmpty(keys)) return new GridFinishedFuture<>(Collections.emptyMap()); - try { - validateTxMode(cacheCtx); - } - catch (IgniteCheckedException e) { - return new GridFinishedFuture(e); - } - if (cacheCtx.mvccEnabled() && !isOperationAllowed(true)) return txTypeMismatchFinishFuture(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java index ffb062daf3a76..e4faf704d186a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java @@ -670,6 +670,26 @@ public static GridNearTxLocal checkActive(GridNearTxLocal tx) { * @return Currently started user transaction, or {@code null} if none started. */ @Nullable public static GridNearTxLocal tx(GridKernalContext ctx, @Nullable GridCacheVersion txId) { + try { + return currentTx(ctx, txId); + } + catch (UnsupportedTxModeException e) { + throw new IgniteSQLException(e.getMessage(), IgniteQueryErrorCode.UNSUPPORTED_OPERATION); + } + catch (NonMvccTransactionException e) { + throw new IgniteSQLException(e.getMessage(), IgniteQueryErrorCode.TRANSACTION_TYPE_MISMATCH); + } + } + + /** + * @param ctx Grid kernal context. + * @param txId Transaction ID. + * @return Currently started user transaction, or {@code null} if none started. + * @throws UnsupportedTxModeException If transaction mode is not supported when MVCC is enabled. + * @throws NonMvccTransactionException If started transaction spans non MVCC caches. + */ + @Nullable public static GridNearTxLocal currentTx(GridKernalContext ctx, + @Nullable GridCacheVersion txId) throws UnsupportedTxModeException, NonMvccTransactionException { IgniteTxManager tm = ctx.cache().context().tm(); IgniteInternalTx tx0 = txId == null ? tm.tx() : tm.tx(txId); @@ -680,23 +700,19 @@ public static GridNearTxLocal checkActive(GridNearTxLocal tx) { if (!tx.pessimistic() || !tx.repeatableRead()) { tx.setRollbackOnly(); - throw new IgniteSQLException("Only pessimistic repeatable read transactions are supported at the moment.", - IgniteQueryErrorCode.UNSUPPORTED_OPERATION); - + throw new UnsupportedTxModeException(); } if (!tx.isOperationAllowed(true)) { tx.setRollbackOnly(); - throw new IgniteSQLException("SQL queries and cache operations " + - "may not be used in the same transaction.", IgniteQueryErrorCode.TRANSACTION_TYPE_MISMATCH); + throw new NonMvccTransactionException(); } } return tx; } - /** * @param ctx Grid kernal context. * @param timeout Transaction timeout. @@ -918,4 +934,24 @@ private static class GetNewVersion implements MvccClosure { return newMvccCrd == MVCC_CRD_COUNTER_NA ? null : mvccVersion(newMvccCrd, newMvccCntr, newMvccOpCntr); } } + + /** */ + public static class UnsupportedTxModeException extends IgniteCheckedException { + /** */ + private static final long serialVersionUID = 0L; + /** */ + private UnsupportedTxModeException() { + super("Only pessimistic repeatable read transactions are supported when MVCC is enabled."); + } + } + + /** */ + public static class NonMvccTransactionException extends IgniteCheckedException { + /** */ + private static final long serialVersionUID = 0L; + /** */ + private NonMvccTransactionException() { + super("Operations on MVCC caches are not permitted in transactions spanning non MVCC caches."); + } + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUnsupportedTxModesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUnsupportedTxModesTest.java new file mode 100644 index 0000000000000..bd82405e2fe33 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUnsupportedTxModesTest.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.mvcc; + +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheEntryProcessor; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionException; +import org.apache.ignite.transactions.TransactionIsolation; + +import static java.util.Collections.singleton; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; +import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; + +/** */ +public class MvccUnsupportedTxModesTest extends GridCommonAbstractTest { + /** */ + private static IgniteCache cache; + /** */ + private static final CacheEntryProcessor testEntryProcessor = (entry, arguments) -> null; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + IgniteEx ign = startGrid(0); + + cache = ign.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAtomicityMode(TRANSACTIONAL_SNAPSHOT)); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + + /** */ + public void testGetAndPutIfAbsent() { + checkOperation(() -> cache.getAndPutIfAbsent(1, 1)); + } + + /** */ + public void testGetAndPutIfAbsentAsync() { + checkOperation(() -> cache.getAndPutIfAbsentAsync(1, 1)); + } + + /** */ + public void testGet() { + checkOperation(() -> cache.get(1)); + } + + /** */ + public void testGetAsync() { + checkOperation(() -> cache.getAsync(1)); + } + + /** */ + public void testGetEntry() { + checkOperation(() -> cache.getEntry(1)); + } + + /** */ + public void testGetEntryAsync() { + checkOperation(() -> cache.getEntryAsync(1)); + } + + /** */ + public void testGetAll() { + checkOperation(() -> cache.getAll(singleton(1))); + } + + /** */ + public void testGetAllAsync() { + checkOperation(() -> cache.getAllAsync(singleton(1))); + } + + /** */ + public void testGetEntries() { + checkOperation(() -> cache.getEntries(singleton(1))); + } + + /** */ + public void testGetEntriesAsync() { + checkOperation(() -> cache.getEntriesAsync(singleton(1))); + } + + /** */ + public void testContainsKey() { + checkOperation(() -> cache.containsKey(1)); + } + + /** */ + public void testContainsKeyAsync() { + checkOperation(() -> cache.containsKeyAsync(1)); + } + + /** */ + public void testContainsKeys() { + checkOperation(() -> cache.containsKeys(singleton(1))); + } + + /** */ + public void testContainsKeysAsync() { + checkOperation(() -> cache.containsKeysAsync(singleton(1))); + } + + /** */ + public void testPut() { + checkOperation(() -> cache.put(1, 1)); + } + + /** */ + public void testPutAsync() { + checkOperation(() -> cache.putAsync(1, 1)); + } + + /** */ + public void testGetAndPut() { + checkOperation(() -> cache.getAndPut(1, 1)); + } + + /** */ + public void testGetAndPutAsync() { + checkOperation(() -> cache.getAndPutAsync(1, 1)); + } + + /** */ + public void testPutAll() { + checkOperation(() -> cache.putAll(ImmutableMap.of(1, 1))); + } + + /** */ + public void testPutAllAsync() { + checkOperation(() -> cache.putAllAsync(ImmutableMap.of(1, 1))); + } + + /** */ + public void testPutIfAbsent() { + checkOperation(() -> cache.putIfAbsent(1, 1)); + } + + /** */ + public void testPutIfAbsentAsync() { + checkOperation(() -> cache.putIfAbsentAsync(1, 1)); + } + + /** */ + public void testRemove1() { + checkOperation(() -> cache.remove(1)); + } + + /** */ + public void testRemoveAsync1() { + checkOperation(() -> cache.removeAsync(1)); + } + + /** */ + public void testRemove2() { + checkOperation(() -> cache.remove(1, 1)); + } + + /** */ + public void testRemoveAsync2() { + checkOperation(() -> cache.removeAsync(1, 1)); + } + + /** */ + public void testGetAndRemove() { + checkOperation(() -> cache.getAndRemove(1)); + } + + /** */ + public void testGetAndRemoveAsync() { + checkOperation(() -> cache.getAndRemoveAsync(1)); + } + + /** */ + public void testReplace1() { + checkOperation(() -> cache.replace(1, 1, 1)); + } + + /** */ + public void testReplaceAsync1() { + checkOperation(() -> cache.replaceAsync(1, 1, 1)); + } + + /** */ + public void testReplace2() { + checkOperation(() -> cache.replace(1, 1)); + } + + /** */ + public void testReplaceAsync2() { + checkOperation(() -> cache.replaceAsync(1, 1)); + } + + /** */ + public void testGetAndReplace() { + checkOperation(() -> cache.getAndReplace(1, 1)); + } + + /** */ + public void testGetAndReplaceAsync() { + checkOperation(() -> cache.getAndReplaceAsync(1, 1)); + } + + /** */ + public void testRemoveAll1() { + checkOperation(() -> cache.removeAll(singleton(1))); + } + + /** */ + public void testRemoveAllAsync1() { + checkOperation(() -> cache.removeAllAsync(singleton(1))); + } + + /** */ + public void testInvoke1() { + checkOperation(() -> cache.invoke(1, testEntryProcessor)); + } + + /** */ + public void testInvokeAsync1() { + checkOperation(() -> cache.invokeAsync(1, testEntryProcessor)); + } + + /** */ + public void testInvoke2() { + checkOperation(() -> cache.invoke(1, testEntryProcessor)); + } + + /** */ + public void testInvokeAsync2() { + checkOperation(() -> cache.invokeAsync(1, testEntryProcessor)); + } + + /** */ + public void testInvokeAll1() { + checkOperation(() -> cache.invokeAll(singleton(1), testEntryProcessor)); + } + + /** */ + public void testInvokeAllAsync1() { + checkOperation(() -> cache.invokeAllAsync(singleton(1), testEntryProcessor)); + } + + /** */ + public void testInvokeAll2() { + checkOperation(() -> cache.invokeAll(singleton(1), testEntryProcessor)); + } + + /** */ + public void testInvokeAllAsync2() { + checkOperation(() -> cache.invokeAllAsync(singleton(1), testEntryProcessor)); + } + + /** */ + public void testInvokeAll3() { + checkOperation(() -> cache.invokeAll(Collections.singletonMap(1, testEntryProcessor))); + } + + /** */ + public void testInvokeAllAsync3() { + checkOperation(() -> cache.invokeAllAsync(Collections.singletonMap(1, testEntryProcessor))); + } + + /** + * @param action Action. + */ + private void checkOperation(Runnable action) { + assertNotSupportedInTx(action, OPTIMISTIC, READ_COMMITTED); + assertNotSupportedInTx(action, OPTIMISTIC, REPEATABLE_READ); + assertNotSupportedInTx(action, OPTIMISTIC, SERIALIZABLE); + assertNotSupportedInTx(action, PESSIMISTIC, READ_COMMITTED); + assertNotSupportedInTx(action, PESSIMISTIC, SERIALIZABLE); + } + + /** */ + private void assertNotSupportedInTx(Runnable action, TransactionConcurrency conc, TransactionIsolation iso) { + try (Transaction ignored = grid(0).transactions().txStart(conc, iso)) { + action.run(); + + fail("Action failure is expected."); + } + catch (TransactionException e) { + assertEquals("Only pessimistic repeatable read transactions are supported when MVCC is enabled.", e.getMessage()); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java index 0c086f50ad284..21b4ae398f226 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTransactionsTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccVacuumTest; import org.apache.ignite.internal.processors.cache.mvcc.MvccCachePeekTest; +import org.apache.ignite.internal.processors.cache.mvcc.MvccUnsupportedTxModesTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; @@ -61,6 +62,8 @@ public static TestSuite suite() { suite.addTestSuite(CacheMvccRemoteTxOnNearNodeStartTest.class); + suite.addTestSuite(MvccUnsupportedTxModesTest.class); + suite.addTestSuite(MvccCachePeekTest.class); suite.addTestSuite(MvccIgniteCacheTxPeekModesTest.class); From cb149a224a85f6a22943b18f414b64aec681d3f6 Mon Sep 17 00:00:00 2001 From: amashenkov Date: Wed, 21 Nov 2018 14:41:26 +0300 Subject: [PATCH 159/403] IGNITE-10260: MVCC: Fix IgniteCache.getAndPutIfAbsent operation. This closes #5412. (cherry picked from commit 529b6de6a46b276a1d007bcefd9e41734911d312) --- .../processors/cache/GridCacheMapEntry.java | 6 ++++ .../dht/GridDhtTxEnlistFuture.java | 13 ++++----- .../tree/mvcc/data/MvccUpdateDataRow.java | 11 ++++---- .../cache/mvcc/CacheMvccTransactionsTest.java | 28 +++++++++++++++++++ .../CacheMvccSqlTxQueriesAbstractTest.java | 2 -- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index bd2d5afc3dfc7..104e8eb0e4dfc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1161,6 +1161,9 @@ else if (res.resultType() == ResultType.FILTERED) { updRes.filtered(true); + if(retVal) + updRes.prevValue(res.oldValue()); + return updRes; } else if(noCreate && !invoke && res.resultType() == ResultType.PREV_NULL) @@ -5485,6 +5488,9 @@ else if (res.resultType() == ResultType.FILTERED) { updRes.filtered(true); + if(needVal) + updRes.prevValue(res.oldValue()); + resFut.onDone(updRes); return; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java index 7719638a89a78..60644cd8e351a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java @@ -119,19 +119,16 @@ public GridDhtTxEnlistFuture(UUID nearNodeId, res.success(txRes.success()); - if(txRes.invokeResult() != null) + if(txRes.invokeResult() != null) { res.invokeResult(true); - if (needRes && txRes.success()) { CacheInvokeResult invokeRes = txRes.invokeResult(); - if (invokeRes != null) { - if(invokeRes.result() != null || invokeRes.error() != null) - res.addEntryProcessResult(cctx, key, null, invokeRes.result(), invokeRes.error(), cctx.keepBinary()); - } - else - res.set(cctx, txRes.prevValue(), txRes.success(), true); + if (invokeRes.result() != null || invokeRes.error() != null) + res.addEntryProcessResult(cctx, key, null, invokeRes.result(), invokeRes.error(), cctx.keepBinary()); } + else if (needRes) + res.set(cctx, txRes.prevValue(), txRes.success(), true); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java index fd1de50062c83..06a113b9dd542 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java @@ -273,8 +273,11 @@ && isActive(cctx, lockCrd, lockCntr, mvccSnapshot)) { else { // Actually, full row can be omitted for replace(k,newval) and putIfAbsent, but // operation context is not available here and full row required if filter is set. - if (res == ResultType.PREV_NOT_NULL && (isFlagsSet(NEED_PREV_VALUE) || filter != null)) - oldRow = tree.getRow(io, pageAddr, idx, RowData.FULL); + if (res == ResultType.PREV_NOT_NULL && (isFlagsSet(NEED_PREV_VALUE) || filter != null)) { + oldRow = tree.getRow(io, pageAddr, idx, RowData.NO_KEY); + + oldRow.key(key); + } else oldRow = row; } @@ -581,9 +584,7 @@ public List cleanupRows() { /** {@inheritDoc} */ @Override public CacheObject oldValue() { - assert oldRow != null; - - return oldRow.value(); + return oldRow == null ? null : oldRow.value(); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java index ca3c09f013731..631db095dc79a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java @@ -201,6 +201,34 @@ public void testImplicitTxOps() throws Exception { val = (Integer)checkAndGet(false, cache, key, SCAN, GET); assertNull(val); + + val = cache.getAndPutIfAbsent(key, 1); + + assertNull(val); + + val = (Integer)checkAndGet(false, cache, key, SCAN, GET); + + assertEquals((Integer)1, val); + + val = cache.getAndPutIfAbsent(key, 1); + + assertEquals((Integer)1, val); + + val = (Integer)checkAndGet(false, cache, key, SCAN, GET); + + assertEquals((Integer)1, val); + + assertFalse(cache.remove(key, 2)); + + val = (Integer)checkAndGet(false, cache, key, SCAN, GET); + + assertEquals((Integer)1, val); + + cache.remove(key, 1); + + val = (Integer)checkAndGet(false, cache, key, SCAN, GET); + + assertNull(val); } } catch (Exception e) { diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java index b71dd261d4061..62cd5b5e499e6 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java @@ -228,8 +228,6 @@ public void testAccountsTxDmlSql_WithRemoves_ClientServer_Backups2() throws Exce * @throws Exception If failed. */ public void testAccountsTxDmlSql_ClientServer_Backups2_Persistence() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-9292"); - persistence = true; testAccountsTxDmlSql_ClientServer_Backups2(); From 07d397c2fa8082d4b687bcdb9e445d9d39e3d4a4 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Wed, 21 Nov 2018 15:17:52 +0300 Subject: [PATCH 160/403] IGNITE-9928: MVCC TX: Late affinity assignment support. This closes #5057. (cherry picked from commit a8fc7af015febee09b70846277f1170b64bce74e) --- .../cache/IgniteCacheOffheapManager.java | 5 + .../cache/IgniteCacheOffheapManagerImpl.java | 18 ++ .../distributed/dht/GridDhtCacheEntry.java | 8 +- .../dht/GridDhtTxAbstractEnlistFuture.java | 169 +++++++------ .../GridDhtPartitionsExchangeFuture.java | 31 +-- .../dht/topology/GridDhtLocalPartition.java | 2 +- .../near/GridNearTxEnlistFuture.java | 22 +- .../GridNearTxQueryResultsEnlistFuture.java | 21 +- .../persistence/GridCacheOffheapManager.java | 12 + .../cache/persistence/tree/BPlusTree.java | 31 +++ .../cache/transactions/IgniteTxAdapter.java | 46 +++- .../cache/transactions/IgniteTxHandler.java | 239 ++++++++++-------- .../cache/mvcc/CacheMvccAbstractTest.java | 19 +- .../junits/GridAbstractTest.java | 2 +- .../mvcc/CacheMvccBackupsAbstractTest.java | 2 + ...PartitionedSqlCoordinatorFailoverTest.java | 49 +++- 16 files changed, 437 insertions(+), 239 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index db09a897dc24e..cdc44a55931b5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -674,6 +674,11 @@ interface CacheDataStore { */ long fullSize(); + /** + * @return {@code True} if there are no items in the store. + */ + boolean isEmpty(); + /** * Updates size metric for particular cache. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 4a64aa0b3ace4..c00dd1ace1408 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -1534,6 +1534,24 @@ void decrementSize(int cacheId) { return storageSize.get(); } + /** + * @return {@code True} if there are no items in the store. + */ + @Override public boolean isEmpty() { + try { + /* + * TODO https://issues.apache.org/jira/browse/IGNITE-10082 + * Using of counters is cheaper than tree operations. Return size checking after the ticked is resolved. + */ + return grp.mvccEnabled() ? dataTree.isEmpty() : storageSize.get() == 0; + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to perform operation.", e); + + return false; + } + } + /** {@inheritDoc} */ @Override public void updateSize(int cacheId, long delta) { storageSize.addAndGet(delta); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java index 8d029b20beadb..90841523cabb6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java @@ -649,10 +649,10 @@ public boolean clearInternal( ']'); } - if (cctx.mvccEnabled()) - cctx.offheap().mvccRemoveAll(this); - else - removeValue(); + if (cctx.mvccEnabled()) + cctx.offheap().mvccRemoveAll(this); + else + removeValue(); // Give to GC. update(null, 0L, 0L, ver, true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java index 9949930b16bd2..62084dca2f480 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache.distributed.dht; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -386,12 +385,20 @@ private void continueLoop(boolean ignoreCntr) { try { while (true) { + int curPart = -1; + List backups = null; + while (hasNext0()) { Object cur = next0(); - KeyCacheObject key = cctx.toCacheKeyObject(op.isDeleteOrLock() ? cur : ((IgniteBiTuple)cur).getKey()); + KeyCacheObject key = toKey(op, cur); + + if (curPart != key.partition()) + backups = backupNodes(curPart = key.partition()); + + assert backups != null; - if (!ensureFreeSlot(key)) { + if (!ensureFreeSlot(key, backups)) { // Can't advance further at the moment. peek = cur; @@ -442,7 +449,7 @@ private void continueLoop(boolean ignoreCntr) { cctx.localNodeId(), topVer, mvccSnapshot, - isMoving(key.partition()), + isMoving(key.partition(), backups), needOldVal, filter, needResult()); @@ -463,7 +470,7 @@ private void continueLoop(boolean ignoreCntr) { topVer, mvccSnapshot, op.cacheOperation(), - isMoving(key.partition()), + isMoving(key.partition(), backups), op.noCreate(), needOldVal, filter, @@ -502,13 +509,14 @@ private void continueLoop(boolean ignoreCntr) { res = updateFut.get(); else { GridDhtCacheEntry entry0 = entry; + List backups0 = backups; it.beforeDetach(); updateFut.listen(new CI1>() { @Override public void apply(IgniteInternalFuture fut) { try { - processEntry(entry0, op, fut.get(), val0); + processEntry(entry0, op, fut.get(), val0, backups0); continueLoop(true); } @@ -523,7 +531,7 @@ private void continueLoop(boolean ignoreCntr) { } } - processEntry(entry, op, res, val0); + processEntry(entry, op, res, val0, backups); } if (!hasNext0()) { @@ -591,6 +599,16 @@ private boolean hasNext0() { return peek != FINISHED; } + /** */ + private KeyCacheObject toKey(EnlistOperation op, Object cur) { + KeyCacheObject key = cctx.toCacheKeyObject(op.isDeleteOrLock() ? cur : ((IgniteBiTuple)cur).getKey()); + + if (key.partition() == -1) + key.partition(cctx.affinity().partition(key)); + + return key; + } + /** * @return {@code True} if in-flight batches map is empty. */ @@ -611,41 +629,40 @@ private boolean noPendingRequests() { * @param op Operation. * @param updRes Update result. * @param val New value. + * @param backups Backup nodes * @throws IgniteCheckedException If failed. */ private void processEntry(GridDhtCacheEntry entry, EnlistOperation op, - GridCacheUpdateTxResult updRes, Message val) throws IgniteCheckedException { + GridCacheUpdateTxResult updRes, Message val, List backups) throws IgniteCheckedException { checkCompleted(); assert updRes != null && updRes.updateFuture() == null; onEntryProcessed(entry.key(), updRes); - if (!updRes.success()) + if (!updRes.success() + || updRes.filtered() + || op == EnlistOperation.LOCK) return; - if (!updRes.filtered()) - cctx.shared().mvccCaching().addEnlisted(entry.key(), updRes.newValue(), 0, 0, lockVer, - updRes.oldValue(), tx.local(), tx.topologyVersion(), mvccSnapshot, cctx.cacheId(), tx, null, -1); + cctx.shared().mvccCaching().addEnlisted(entry.key(), updRes.newValue(), 0, 0, lockVer, + updRes.oldValue(), tx.local(), tx.topologyVersion(), mvccSnapshot, cctx.cacheId(), tx, null, -1); - if (op != EnlistOperation.LOCK) - addToBatch(entry.key(), val, updRes.mvccHistory(), entry.context().cacheId(), op); + addToBatch(entry.key(), val, updRes.mvccHistory(), entry.context().cacheId(), backups); } /** * Adds row to batch. * IMPORTANT: This method should be called from the critical section in {@link this.sendNextBatches()} - * * @param key Key. * @param val Value. * @param hist History rows. * @param cacheId Cache Id. + * @param backups Backup nodes */ private void addToBatch(KeyCacheObject key, Message val, List hist, - int cacheId, EnlistOperation op) throws IgniteCheckedException { - List backups = backupNodes(key); - - int part = cctx.affinity().partition(key); + int cacheId, List backups) throws IgniteCheckedException { + int part = key.partition(); tx.touchPartition(cacheId, part); @@ -659,7 +676,7 @@ private void addToBatch(KeyCacheObject key, Message val, List backups) { if (F.isEmpty(batches) || F.isEmpty(pending)) return true; + int part = key.partition(); + // Check possibility of adding to batch and sending. - for (ClusterNode node : backupNodes(key)) { - if (skipNearNodeUpdates && node.id().equals(nearNodeId) && !isMoving(node, key.partition())) + for (ClusterNode node : backups) { + if (skipNearLocalUpdate(node, isMoving(node, part))) continue; Batch batch = batches.get(node.id()); @@ -816,16 +834,14 @@ private void sendBatch(Batch batch) throws IgniteCheckedException { if (newRemoteTx(node)) addNewRemoteTxNode(node); - if (!firstReqSent.contains(node)) { - firstReqSent.add(node); - + if (firstReqSent.add(node)) { // If this is a first request to this node, send full info. req = new GridDhtTxQueryFirstEnlistRequest(cctx.cacheId(), futId, cctx.localNodeId(), tx.topologyVersionSnapshot(), lockVer, - mvccSnapshot, + mvccSnapshot.withoutActiveTransactions(), tx.remainingTime(), tx.taskNameHash(), nearNodeId, @@ -863,7 +879,13 @@ private void sendBatch(Batch batch) throws IgniteCheckedException { assert prev == null; - cctx.io().send(node, req, cctx.ioPolicy()); + try { + cctx.io().send(node, req, cctx.ioPolicy()); + } + catch (ClusterTopologyCheckedException e) { + // backup node left the grid, will continue. + onNodeLeft(node.id()); + } } /** */ @@ -880,20 +902,21 @@ private synchronized void updateMappings(ClusterNode node) throws IgniteCheckedE mapping.markQueryUpdate(); } + /** */ + private boolean skipNearLocalUpdate(ClusterNode node, boolean moving) { + return skipNearNodeUpdates && node.id().equals(nearNodeId) && !moving; + } + /** - * @param key Key. - * @return Backup nodes for the given key. + * @param part Partition. + * @return Backup nodes for the given partition. */ - @NotNull private List backupNodes(KeyCacheObject key) { - List dhtNodes = cctx.affinity().nodesByKey(key, tx.topologyVersion()); - - assert !dhtNodes.isEmpty() && dhtNodes.get(0).id().equals(cctx.localNodeId()) : - "localNode = " + cctx.localNodeId() + ", dhtNodes = " + dhtNodes; + @NotNull private List backupNodes(int part) { + List nodes = cctx.topology().nodes(part, tx.topologyVersion()); - if (dhtNodes.size() == 1) - return Collections.emptyList(); + assert nodes.size() > 0 && nodes.get(0).isLocal(); - return dhtNodes.subList(1, dhtNodes.size()); + return nodes.subList(1, nodes.size()); } /** @@ -933,31 +956,33 @@ private void checkPartitions(@Nullable int[] parts) throws ClusterTopologyChecke /** * @param part Partition. + * @param backups Backup nodes. * @return {@code true} if the given partition is rebalancing to any backup node. */ - private boolean isMoving(int part) { + private boolean isMoving(int part, List backups) { + Boolean res; + if (movingParts == null) movingParts = new HashMap<>(); - Boolean res = movingParts.get(part); + if ((res = movingParts.get(part)) == null) + movingParts.put(part, res = isMoving0(part, backups)); - if (res != null) - return res; - - List dhtNodes = cctx.affinity().nodesByPartition(part, tx.topologyVersion()); - - for (int i = 1; i < dhtNodes.size(); i++) { - ClusterNode node = dhtNodes.get(i); - if (isMoving(node, part)) { - movingParts.put(part, Boolean.TRUE); + return res == Boolean.TRUE; + } - return true; - } + /** + * @param part Partition. + * @param backups Backup nodes. + * @return {@code true} if the given partition is rebalancing to any backup node. + */ + private Boolean isMoving0(int part, List backups) { + for (ClusterNode node : backups) { + if (isMoving(node, part)) + return Boolean.TRUE; } - movingParts.put(part, Boolean.FALSE); - - return false; + return Boolean.FALSE; } /** @@ -966,9 +991,7 @@ private boolean isMoving(int part) { * @return {@code true} if the given partition is rebalancing to the given node. */ private boolean isMoving(ClusterNode node, int part) { - GridDhtPartitionState partState = cctx.topology().partitionState(node.id(), part); - - return partState != GridDhtPartitionState.OWNING && partState != GridDhtPartitionState.EVICTED; + return cctx.topology().partitionState(node.id(), part) == GridDhtPartitionState.MOVING; } /** */ @@ -1021,23 +1044,17 @@ public void onResult(UUID nodeId, GridDhtTxQueryEnlistResponse res) { /** {@inheritDoc} */ @Override public boolean onNodeLeft(UUID nodeId) { - boolean backupLeft = false; - - Set nodes = tx.lockTransactionNodes(); - - if (!F.isEmpty(nodes)) { - for (ClusterNode node : nodes) { - if (node.id().equals(nodeId)) { - backupLeft = true; - - break; - } - } + try { + if (nearNodeId.equals(nodeId)) + onDone(new ClusterTopologyCheckedException("Requesting node left the grid [nodeId=" + nodeId + ']')); + else if (pending != null && pending.remove(nodeId) != null) + cctx.kernalContext().closure().runLocalSafe(() -> continueLoop(false)); + } + catch (Exception e) { + onDone(e); } - return (backupLeft || nearNodeId.equals(nodeId)) && onDone( - new ClusterTopologyCheckedException((backupLeft ? "Backup" : "Requesting") + - " node left the grid [nodeId=" + nodeId + ']')); + return false; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 3149c4f5d0691..b5e49da1ce712 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3570,27 +3570,24 @@ private void assignPartitionsStates() { * failed to send update counter deltas to backup. */ private void finalizePartitionCounters() { - long time = System.currentTimeMillis(); + int parallelismLvl = cctx.kernalContext().config().getSystemThreadPoolSize(); - try { - int parallelismLvl = cctx.kernalContext().config().getSystemThreadPoolSize(); + // Reserve at least 2 threads for system operations. + parallelismLvl = Math.max(1, parallelismLvl - 2); - // Reserve at least 2 threads for system operations. - parallelismLvl = Math.max(1, parallelismLvl - 2); + long time = System.currentTimeMillis(); - if (parallelismLvl > 1) { - U.doInParallel(parallelismLvl, - cctx.kernalContext().getSystemExecutorService(), - nonLocalCacheGroups(), - grp -> { - grp.topology().finalizeUpdateCounters(); + try { + U.doInParallel( + parallelismLvl, + cctx.kernalContext().getSystemExecutorService(), + nonLocalCacheGroups(), + grp -> { + grp.topology().finalizeUpdateCounters(); - return null; - } - ); - } - else - nonLocalCacheGroups().forEach(grp -> grp.topology().finalizeUpdateCounters()); + return null; + } + ); } catch (IgniteCheckedException e) { throw new IgniteException("Failed to finalize partition counters", e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 2557bb3946781..75375c16faa64 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -352,7 +352,7 @@ public int reservations() { * @return {@code True} if partition is empty. */ public boolean isEmpty() { - return store.fullSize() == 0 && internalSize() == 0; + return store.isEmpty() && internalSize() == 0; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java index 3ddedd04b653b..598ffc353134f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java @@ -210,11 +210,9 @@ private Collection continueLoop(@Nullable UUID nodeId) throws IgniteCheck KeyCacheObject key = cctx.toCacheKeyObject(op.isDeleteOrLock() ? cur : ((IgniteBiTuple)cur).getKey()); - List nodes = cctx.affinity().nodesByKey(key, topVer); + ClusterNode node = cctx.affinity().primaryByKey(key, topVer); - ClusterNode node; - - if (F.isEmpty(nodes) || ((node = nodes.get(0)) == null)) + if (node == null) throw new ClusterTopologyCheckedException("Failed to get primary node " + "[topVer=" + topVer + ", key=" + key + ']'); @@ -241,8 +239,7 @@ else if (batch != null && !batch.node().equals(node)) break; } - batch.add(op.isDeleteOrLock() ? key : cur, - op != EnlistOperation.LOCK && cctx.affinityNode() && (cctx.isReplicated() || nodes.indexOf(cctx.localNode()) > 0)); + batch.add(op.isDeleteOrLock() ? key : cur, !node.isLocal() && isLocalBackup(op, key)); if (batch.size() == batchSize) res = markReady(res, batch); @@ -298,6 +295,16 @@ private boolean hasNext0() { return peek != FINISHED; } + /** */ + private boolean isLocalBackup(EnlistOperation op, KeyCacheObject key) { + if (!cctx.affinityNode() || op == EnlistOperation.LOCK) + return false; + else if (cctx.isReplicated()) + return true; + + return cctx.topology().nodes(key.partition(), tx.topologyVersion()).indexOf(cctx.localNode()) > 0; + } + /** * Add batch to batch collection if it is ready. * @@ -605,6 +612,9 @@ public boolean checkResponse(UUID nodeId, GridNearTxEnlistResponse res, Throwabl assert this.res != null && (this.res.emptyResult() || needRes || this.res.invokeResult() || !this.res.success()); + + tx.hasRemoteLocks(true); + return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java index 9783582c41d46..11ae966b9160e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java @@ -201,11 +201,9 @@ private Collection continueLoop(@Nullable UUID nodeId) throws IgniteCheck KeyCacheObject key = cctx.toCacheKeyObject(op.isDeleteOrLock() ? cur : ((IgniteBiTuple)cur).getKey()); - List nodes = cctx.affinity().nodesByKey(key, topVer); + ClusterNode node = cctx.affinity().primaryByPartition(key.partition(), topVer); - ClusterNode node; - - if (F.isEmpty(nodes) || ((node = nodes.get(0)) == null)) + if (node == null) throw new ClusterTopologyCheckedException("Failed to get primary node " + "[topVer=" + topVer + ", key=" + key + ']'); @@ -230,8 +228,7 @@ else if (batch != null && !batch.node().equals(node)) break; } - batch.add(op.isDeleteOrLock() ? key : cur, - op != EnlistOperation.LOCK && cctx.affinityNode() && (cctx.isReplicated() || nodes.indexOf(cctx.localNode()) > 0)); + batch.add(op.isDeleteOrLock() ? key : cur, !node.isLocal() && isLocalBackup(op, key)); if (batch.size() == batchSize) res = markReady(res, batch); @@ -287,6 +284,16 @@ private boolean hasNext0() { return peek != FINISHED; } + /** */ + private boolean isLocalBackup(EnlistOperation op, KeyCacheObject key) { + if (!cctx.affinityNode() || op == EnlistOperation.LOCK) + return false; + else if (cctx.isReplicated()) + return true; + + return cctx.topology().nodes(key.partition(), tx.topologyVersion()).contains(cctx.localNode()); + } + /** */ private ArrayList markReady(ArrayList batches, Batch batch) { if (!batch.ready()) { @@ -571,6 +578,8 @@ public boolean checkResponse(UUID nodeId, GridNearTxQueryResultsEnlistResponse r RES_UPD.getAndAdd(this, res.result()); + tx.hasRemoteLocks(true); + return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 73cb878baf528..fb287bbcfb6a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -1769,6 +1769,18 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { } } + /** {@inheritDoc} */ + @Override public boolean isEmpty() { + try { + CacheDataStore delegate0 = init0(true); + + return delegate0 == null || delegate0.isEmpty(); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + /** {@inheritDoc} */ @Override public long cacheSize(int cacheId) { try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java index fd7fba63d1ae3..ce9463dc1e78b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java @@ -2097,6 +2097,37 @@ public final int rootLevel() throws IgniteCheckedException { return getRootLevel(); } + /** + * @return {@code True} in case the tree is empty. + * @throws IgniteCheckedException If failed. + */ + public final boolean isEmpty() throws IgniteCheckedException { + checkDestroyed(); + + for (;;) { + TreeMetaData treeMeta = treeMeta(); + + long rootId, rootPage = acquirePage(rootId = treeMeta.rootId); + + long rootAddr = readLock(rootId, rootPage); + + if (rootAddr == 0) { + checkDestroyed(); + + continue; + } + + try { + BPlusIO io = io(rootAddr); + + return io.getCount(rootAddr) == 0; + } + finally { + readUnlock(rootId, rootPage, rootAddr); + } + } + } + /** * Returns number of elements in the tree by scanning pages of the bottom (leaf) level. * Since a concurrent access is permitted, there is no guarantee about diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index 89a30dd356a26..306c580f42d04 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -62,7 +62,9 @@ import org.apache.ignite.internal.processors.cache.GridCacheReturn; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; @@ -1989,21 +1991,51 @@ protected void applyTxSizes() { for (Map.Entry> entry : sizeDeltas.entrySet()) { Integer cacheId = entry.getKey(); - Map partDeltas = entry.getValue(); + Map deltas = entry.getValue(); - assert !F.isEmpty(partDeltas); + assert !F.isEmpty(deltas); GridDhtPartitionTopology top = cctx.cacheContext(cacheId).topology(); - for (Map.Entry e : partDeltas.entrySet()) { - Integer p = e.getKey(); + // Need to reserve on backups only + boolean reserve = dht() && remote(); + + for (Map.Entry e : deltas.entrySet()) { + boolean invalid = false; + int p = e.getKey(); long delta = e.getValue().get(); - GridDhtLocalPartition dhtPart = top.localPartition(p); + try { + GridDhtLocalPartition part = top.localPartition(p); + + if (!reserve || part != null && part.reserve()) { + assert part != null; + + try { + if (part.state() != GridDhtPartitionState.RENTING) + part.dataStore().updateSize(cacheId, delta); + else + invalid = true; + } + finally { + if (reserve) + part.release(); + } + } + else + invalid = true; + } + catch (GridDhtInvalidPartitionException e1) { + invalid = true; + } - assert dhtPart != null; + if (invalid) { + assert reserve; - dhtPart.dataStore().updateSize(cacheId, delta); + if (log.isDebugEnabled()) + log.debug("Trying to apply size delta for invalid partition: " + + "[cacheId=" + cacheId + ", part=" + p + "]"); + } } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 434f839ac5103..6ea03fc78b2f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -25,7 +25,6 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.cluster.ClusterTopologyException; import org.apache.ignite.failure.FailureContext; import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; @@ -34,7 +33,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryInfoCollection; import org.apache.ignite.internal.processors.cache.CacheObject; -import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo; @@ -64,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionUpdateCountersMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture; @@ -1663,8 +1662,6 @@ private void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean committe GridDhtTxRemote tx = ctx.tm().tx(req.version()); if (tx == null) { - assert !req.queryUpdate(); - boolean single = req.last() && req.writes().size() == 1; tx = new GridDhtTxRemote( @@ -1864,117 +1861,132 @@ public void mvccEnlistBatch(GridDhtTxRemote tx, GridCacheContext ctx, EnlistOper int part = ctx.affinity().partition(key); - GridDhtLocalPartition locPart = ctx.topology().localPartition(part, tx.topologyVersion(), false); + try { + GridDhtLocalPartition locPart = ctx.topology().localPartition(part, tx.topologyVersion(), false); - if (locPart == null || !locPart.reserve()) - throw new ClusterTopologyException("Can not reserve partition. Please retry on stable topology."); + if (locPart != null && locPart.reserve()) { + try { + // do not process renting partitions. + if (locPart.state() == GridDhtPartitionState.RENTING) { + tx.addInvalidPartition(ctx, part); - try { - CacheObject val = null; - EntryProcessor entryProc = null; - Object[] invokeArgs = null; + continue; + } - boolean needOldVal = ctx.shared().mvccCaching().continuousQueryListeners(ctx, tx, key) != null; + CacheObject val = null; + EntryProcessor entryProc = null; + Object[] invokeArgs = null; - Message val0 = vals != null ? vals.get(i) : null; + boolean needOldVal = ctx.shared().mvccCaching().continuousQueryListeners(ctx, tx, key) != null; - CacheEntryInfoCollection entries = - val0 instanceof CacheEntryInfoCollection ? (CacheEntryInfoCollection)val0 : null; + Message val0 = vals != null ? vals.get(i) : null; - if (entries == null && !op.isDeleteOrLock() && !op.isInvoke()) - val = (val0 instanceof CacheObject) ? (CacheObject)val0 : null; + CacheEntryInfoCollection entries = + val0 instanceof CacheEntryInfoCollection ? (CacheEntryInfoCollection)val0 : null; - if(entries == null && op.isInvoke()) { - assert val0 instanceof GridInvokeValue; + if (entries == null && !op.isDeleteOrLock() && !op.isInvoke()) + val = (val0 instanceof CacheObject) ? (CacheObject)val0 : null; - GridInvokeValue invokeVal = (GridInvokeValue)val0; + if (entries == null && op.isInvoke()) { + assert val0 instanceof GridInvokeValue; - entryProc = invokeVal.entryProcessor(); - invokeArgs = invokeVal.invokeArgs(); - } + GridInvokeValue invokeVal = (GridInvokeValue)val0; - assert entryProc != null || !op.isInvoke() : "entryProc=" + entryProc + ", op=" + op; + entryProc = invokeVal.entryProcessor(); + invokeArgs = invokeVal.invokeArgs(); + } - GridDhtCacheEntry entry = dht.entryExx(key, tx.topologyVersion()); + assert entryProc != null || !op.isInvoke() : "entryProc=" + entryProc + ", op=" + op; - GridCacheUpdateTxResult updRes; + GridDhtCacheEntry entry = dht.entryExx(key, tx.topologyVersion()); - while (true) { - ctx.shared().database().checkpointReadLock(); + GridCacheUpdateTxResult updRes; - try { - if (entries == null) { - switch (op) { - case DELETE: - updRes = entry.mvccRemove( - tx, - ctx.localNodeId(), - tx.topologyVersion(), - snapshot, - false, - needOldVal, - null, - false); - - break; - - case INSERT: - case TRANSFORM: - case UPSERT: - case UPDATE: - updRes = entry.mvccSet( - tx, + while (true) { + ctx.shared().database().checkpointReadLock(); + + try { + if (entries == null) { + switch (op) { + case DELETE: + updRes = entry.mvccRemove( + tx, + ctx.localNodeId(), + tx.topologyVersion(), + snapshot, + false, + needOldVal, + null, + false); + + break; + + case INSERT: + case TRANSFORM: + case UPSERT: + case UPDATE: + updRes = entry.mvccSet( + tx, + ctx.localNodeId(), + val, + entryProc, + invokeArgs, + 0, + tx.topologyVersion(), + snapshot, + op.cacheOperation(), + false, + false, + needOldVal, + null, + false); + + break; + + default: + throw new IgniteSQLException("Cannot acquire lock for operation [op= " + + op + "]" + "Operation is unsupported at the moment ", + IgniteQueryErrorCode.UNSUPPORTED_OPERATION); + } + } + else { + updRes = entry.mvccUpdateRowsWithPreloadInfo(tx, ctx.localNodeId(), - val, - entryProc, - invokeArgs, - 0, tx.topologyVersion(), - snapshot, + entries.infos(), op.cacheOperation(), - false, - false, - needOldVal, - null, - false); - - break; - - default: - throw new IgniteSQLException("Cannot acquire lock for operation [op= " - + op + "]" + "Operation is unsupported at the moment ", - IgniteQueryErrorCode.UNSUPPORTED_OPERATION); + snapshot, + futId, + batchNum); + } + + break; + } + catch (GridCacheEntryRemovedException ignore) { + entry = dht.entryExx(key); + } + finally { + ctx.shared().database().checkpointReadUnlock(); } - } - else { - updRes = entry.mvccUpdateRowsWithPreloadInfo(tx, - ctx.localNodeId(), - tx.topologyVersion(), - entries.infos(), - op.cacheOperation(), - snapshot, - futId, - batchNum); } - break; - } - catch (GridCacheEntryRemovedException ignore) { - entry = dht.entryExx(key); + if (!updRes.filtered()) + ctx.shared().mvccCaching().addEnlisted(key, updRes.newValue(), 0, 0, tx.xidVersion(), + updRes.oldValue(), tx.local(), tx.topologyVersion(), snapshot, ctx.cacheId(), tx, futId, batchNum); + + assert updRes.updateFuture() == null : "Entry should not be locked on the backup"; } + finally { - ctx.shared().database().checkpointReadUnlock(); + locPart.release(); } } - - if (!updRes.filtered()) - ctx.shared().mvccCaching().addEnlisted(key, updRes.newValue(), 0, 0, tx.xidVersion(), - updRes.oldValue(), tx.local(), tx.topologyVersion(), snapshot, ctx.cacheId(), tx, futId, batchNum); - - assert updRes.updateFuture() == null : "Entry should not be locked on the backup"; + else { + tx.addInvalidPartition(ctx, part); + } } - finally { - locPart.release(); + catch (GridDhtInvalidPartitionException e) { + tx.addInvalidPartition(ctx, e.partition()); } } } @@ -2215,26 +2227,42 @@ public void applyPartitionsUpdatesCounters(Iterable filterUpdateCountersForBackupNode( IgniteInternalTx tx, ClusterNode node) { TxCounters txCntrs = tx.txCounters(false); + Collection updCntrs; - if (txCntrs == null || F.isEmpty(txCntrs.updateCounters())) + if (txCntrs == null || F.isEmpty(updCntrs = txCntrs.updateCounters())) return null; - Collection updCntrs = txCntrs.updateCounters(); - List res = new ArrayList<>(updCntrs.size()); AffinityTopologyVersion top = tx.topologyVersionSnapshot(); for (PartitionUpdateCountersMessage partCntrs : updCntrs) { - GridCacheAffinityManager affinity = ctx.cacheContext(partCntrs.cacheId()).affinity(); + GridDhtPartitionTopology topology = ctx.cacheContext(partCntrs.cacheId()).topology(); PartitionUpdateCountersMessage resCntrs = new PartitionUpdateCountersMessage(partCntrs.cacheId(), partCntrs.size()); for (int i = 0; i < partCntrs.size(); i++) { int part = partCntrs.partition(i); - if (affinity.backupByPartition(node, part, top)) + if (topology.nodes(part, top).indexOf(node) > 0) resCntrs.add(part, partCntrs.initialCounter(i), partCntrs.updatesCount(i)); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java index 5b3234762d391..aa4ec26b2116f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java @@ -945,6 +945,8 @@ protected void putAllGetAll( tx.commit(); + v++; + first = false; } @@ -952,10 +954,8 @@ protected void putAllGetAll( Map res = readAllByMode(cache.cache, keys, readMode, INTEGER_CODEC); for (Integer k : keys) - assertEquals("key=" + k, v, (Object)res.get(k)); + assertEquals("key=" + k, v - 1, (Object)res.get(k)); } - - v++; } catch (Exception e) { handleTxException(e); @@ -1367,6 +1367,13 @@ final void readWriteTest( } }, readers, "reader"); + GridTestUtils.runAsync(() -> { + while (System.currentTimeMillis() < stopTime) + doSleep(1000); + + stop.set(true); + }); + while (System.currentTimeMillis() < stopTime && !stop.get()) { Thread.sleep(1000); @@ -1408,8 +1415,10 @@ final void readWriteTest( Ignite srv = startGrid(idx); + cache0 = new TestCache(srv.cache(DEFAULT_CACHE_NAME)); + synchronized (caches) { - caches.set(idx, new TestCache(srv.cache(DEFAULT_CACHE_NAME))); + caches.set(idx, cache0); } awaitPartitionMapExchange(); @@ -1423,8 +1432,6 @@ final void readWriteTest( } } - stop.set(true); - Exception ex = null; try { diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index 064080bcd5e5a..0be0d252b07eb 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -2311,7 +2311,7 @@ public static void doSleep(long millis) { U.sleep(millis); } catch (Exception e) { - throw new IgniteException(); + throw new IgniteException(e); } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java index 998cb766668c7..894e4bb56eb5d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java @@ -182,6 +182,8 @@ public void testBackupsCoherenceSimple() throws Exception { * @throws Exception If failed. */ public void testBackupsCoherenceWithLargeOperations() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10104"); + disableScheduledVacuum = true; ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 1, 10) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedSqlCoordinatorFailoverTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedSqlCoordinatorFailoverTest.java index b347497b5b2b2..eee270c7e35c8 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedSqlCoordinatorFailoverTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedSqlCoordinatorFailoverTest.java @@ -22,7 +22,6 @@ import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.DML; -import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.PUT; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; @@ -61,14 +60,6 @@ public void testPutAllGetAll_ClientServer_Backups3_RestartCoordinator_ScanDml() new InitIndexing(Integer.class, Integer.class), SCAN, DML); } - /** - * @throws Exception If failed. - */ - public void testPutAllGetAll_ClientServer_Backups1_Restart_Scan() throws Exception { - // TODO add tests with RESTART_RND_SRV https://issues.apache.org/jira/browse/IGNITE-9928 - putAllGetAll(RestartMode.RESTART_RND_SRV, 4, 2, 2, 64, /*new InitIndexing(Integer.class, Integer.class)*/ null, SCAN, PUT); - } - /** * @throws Exception If failed. */ @@ -97,6 +88,46 @@ public void testPutAllGetAll_ClientServer_Backups1_RestartCoordinator_SqlDml() t new InitIndexing(Integer.class, Integer.class), SQL, DML); } + /** + * @throws Exception If failed. + */ + public void testPutAllGetAll_ClientServer_Backups1_RestartRandomSrv_SqlDml() throws Exception { + putAllGetAll(RestartMode.RESTART_RND_SRV, 3, 1, 1, DFLT_PARTITION_COUNT, + new InitIndexing(Integer.class, Integer.class), SQL, DML); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllGetAll_ClientServer_Backups2_RestartRandomSrv_SqlDml() throws Exception { + putAllGetAll(RestartMode.RESTART_RND_SRV, 4, 1, 2, DFLT_PARTITION_COUNT, + new InitIndexing(Integer.class, Integer.class), SQL, DML); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllGetAll_Server_Backups2_RestartRandomSrv_SqlDml() throws Exception { + putAllGetAll(RestartMode.RESTART_RND_SRV, 4, 0, 2, DFLT_PARTITION_COUNT, + new InitIndexing(Integer.class, Integer.class), SQL, DML); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllGetAll_Server_Backups1_SinglePartition_RestartRandomSrv_SqlDml() throws Exception { + putAllGetAll(RestartMode.RESTART_RND_SRV, 4, 0, 1, 1, + new InitIndexing(Integer.class, Integer.class), SQL, DML); + } + + /** + * @throws Exception If failed. + */ + public void testPutAllGetAll_ClientServer_Backups1_SinglePartition_RestartRandomSrv_SqlDml() throws Exception { + putAllGetAll(RestartMode.RESTART_RND_SRV, 3, 1, 1, 1, + new InitIndexing(Integer.class, Integer.class), SQL, DML); + } + /** * @throws Exception If failed. */ From 393d8fb978dd000d8b0c452b4d1a32ab295965ba Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 21 Nov 2018 18:19:32 +0300 Subject: [PATCH 161/403] IGNITE-10226 Fixed wrong partition state recovery - Fixes #5396. Signed-off-by: Pavel Kovalenko (cherry picked from commit ceba214) --- .../dht/topology/GridDhtLocalPartition.java | 29 ++- .../GridDhtPartitionTopologyImpl.java | 6 +- .../GridCacheDatabaseSharedManager.java | 2 +- .../IgnitePdsPartitionsStateRecoveryTest.java | 170 ++++++++++++++++++ .../testsuites/IgnitePdsTestSuite2.java | 3 + 5 files changed, 202 insertions(+), 8 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionsStateRecoveryTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 75375c16faa64..ca3a154c782c8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -32,6 +32,8 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.NodeStoppingException; @@ -172,11 +174,15 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements * @param ctx Context. * @param grp Cache group. * @param id Partition ID. + * @param recovery Flag indicates that partition is created during recovery phase. */ @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor") - public GridDhtLocalPartition(GridCacheSharedContext ctx, - CacheGroupContext grp, - int id) { + public GridDhtLocalPartition( + GridCacheSharedContext ctx, + CacheGroupContext grp, + int id, + boolean recovery + ) { super(ENTRY_FACTORY); this.id = id; @@ -213,7 +219,7 @@ public GridDhtLocalPartition(GridCacheSharedContext ctx, store = grp.offheap().createCacheDataStore(id); // Log partition creation for further crash recovery purposes. - if (grp.walEnabled()) + if (grp.walEnabled() && !recovery) ctx.wal().log(new PartitionMetaStateRecord(grp.groupId(), id, state(), updateCounter())); // Inject row cache cleaner on store creation @@ -560,6 +566,8 @@ private boolean casState(long state, GridDhtPartitionState toState) { } catch (IgniteCheckedException e) { U.error(log, "Failed to log partition state change to WAL.", e); + + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); } if (log.isDebugEnabled()) @@ -974,6 +982,19 @@ public long nextUpdateCounter(int cacheId, AffinityTopologyVersion topVer, boole if (grp.sharedGroup()) grp.onPartitionCounterUpdate(cacheId, id, primaryCntr != null ? primaryCntr : nextCntr, topVer, primary); + // This is first update in partition, we should log partition state information for further crash recovery. + if (nextCntr == 1) { + if (grp.persistenceEnabled() && grp.walEnabled()) + try { + ctx.wal().log(new PartitionMetaStateRecord(grp.groupId(), id, state(), 0)); + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to log partition state snapshot to WAL.", e); + + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } + } + return nextCntr; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 0e2125bba2392..c32a1de2ef9bf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -864,7 +864,7 @@ private GridDhtLocalPartition getOrCreatePartition(int p) { if (loc != null) loc.awaitDestroy(); - locParts.set(p, loc = new GridDhtLocalPartition(ctx, grp, p)); + locParts.set(p, loc = new GridDhtLocalPartition(ctx, grp, p, false)); long updCntr = cntrMap.updateCounter(p); @@ -895,7 +895,7 @@ private GridDhtLocalPartition getOrCreatePartition(int p) { if (part != null && part.state() != EVICTED) return part; - part = new GridDhtLocalPartition(ctx, grp, p); + part = new GridDhtLocalPartition(ctx, grp, p, true); locParts.set(p, part); @@ -970,7 +970,7 @@ else if (loc != null && state == RENTING && !showRenting) { "[grp=" + grp.cacheOrGroupName() + ", part=" + p + ", topVer=" + topVer + ", this.topVer=" + this.readyTopVer + ']'); - locParts.set(p, loc = new GridDhtLocalPartition(ctx, grp, p)); + locParts.set(p, loc = new GridDhtLocalPartition(ctx, grp, p, false)); this.updateSeq.incrementAndGet(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index bf64d8412a9b1..a2afe02fda967 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -2982,7 +2982,7 @@ protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log) while (!isCancelled()) { waitCheckpointEvent(); - if (skipCheckpointOnNodeStop && (isStopping() || shutdownNow)) { + if (skipCheckpointOnNodeStop && (isCancelled() || shutdownNow)) { if (log.isInfoEnabled()) log.warning("Skipping last checkpoint because node is stopping."); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionsStateRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionsStateRecoveryTest.java new file mode 100644 index 0000000000000..112fc59ecc1ba --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionsStateRecoveryTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence; + +import java.util.Arrays; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheRebalanceMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Assert; + +/** + * + */ +public class IgnitePdsPartitionsStateRecoveryTest extends GridCommonAbstractTest { + /** Cache name. */ + private static final String CACHE = "cache"; + + /** Partitions count. */ + private static final int PARTS_CNT = 32; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setConsistentId(igniteInstanceName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setWalMode(WALMode.LOG_ONLY) + .setWalSegmentSize(16 * 1024 * 1024) + .setCheckpointFrequency(20 * 60 * 1000) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setMaxSize(512 * 1024 * 1024) + .setPersistenceEnabled(true) + ); + + cfg.setDataStorageConfiguration(dsCfg); + + CacheConfiguration ccfg = new CacheConfiguration<>(CACHE) + .setBackups(0) + .setRebalanceMode(CacheRebalanceMode.NONE) // Disable rebalance to prevent owning MOVING partitions. + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT)); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, "true"); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP); + } + + /** + * Test checks that partition state is recovered properly if last checkpoint was skipped and there are logical updates to apply. + * + * @throws Exception If failed. + */ + public void testPartitionsStateConsistencyAfterRecovery() throws Exception { + IgniteEx ignite = startGrid(0); + + ignite.cluster().active(true); + + IgniteCache cache = ignite.getOrCreateCache(CACHE); + + for (int key = 0; key < 4096; key++) + cache.put(key, key); + + forceCheckpoint(); + + for (int key = 0; key < 4096; key++) { + int[] payload = new int[4096]; + Arrays.fill(payload, key); + + cache.put(key, payload); + } + + GridDhtPartitionTopology topology = ignite.cachex(CACHE).context().topology(); + + Assert.assertFalse(topology.hasMovingPartitions()); + + log.info("Stopping grid..."); + + stopGrid(0); + + ignite = startGrid(0); + + awaitPartitionMapExchange(); + + topology = ignite.cachex(CACHE).context().topology(); + + Assert.assertFalse("Node restored moving partitions after join to topology.", topology.hasMovingPartitions()); + } + + /** + * Test checks that partition state is recovered properly if only logical updates exist. + * + * @throws Exception If failed. + */ + public void testPartitionsStateConsistencyAfterRecoveryNoCheckpoints() throws Exception { + IgniteEx ignite = startGrid(0); + + ignite.cluster().active(true); + + IgniteCache cache = ignite.getOrCreateCache(CACHE); + + forceCheckpoint(); + + for (int key = 0; key < 4096; key++) { + int[] payload = new int[4096]; + Arrays.fill(payload, key); + + cache.put(key, payload); + } + + GridDhtPartitionTopology topology = ignite.cachex(CACHE).context().topology(); + + Assert.assertFalse(topology.hasMovingPartitions()); + + log.info("Stopping grid..."); + + stopGrid(0); + + ignite = startGrid(0); + + awaitPartitionMapExchange(); + + topology = ignite.cachex(CACHE).context().topology(); + + Assert.assertFalse("Node restored moving partitions after join to topology.", topology.hasMovingPartitions()); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index 75a8af755549c..c12f515d461b6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionFilesDestroyTest; +import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionsStateRecoveryTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest; import org.apache.ignite.internal.processors.cache.persistence.IgniteRebalanceScheduleResendPartitionsTest; import org.apache.ignite.internal.processors.cache.persistence.LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest; @@ -194,5 +195,7 @@ public static void addRealPageStoreTests(TestSuite suite) { suite.addTestSuite(WalRolloverTypesTest.class); suite.addTestSuite(FsyncWalRolloverDoesNotBlockTest.class); + + suite.addTestSuite(IgnitePdsPartitionsStateRecoveryTest.class); } } From 56f516319ded780ac6537133a1ff900ca7405c9e Mon Sep 17 00:00:00 2001 From: Andrey Kuznetsov Date: Wed, 21 Nov 2018 21:51:29 +0300 Subject: [PATCH 162/403] IGNITE-9695 Add a way to prevent WAL disabling in WalStateManager - Fixes #4835. Signed-off-by: Ivan Rakov (cherry picked from commit d81acdc) --- .../processors/cache/WalStateManager.java | 24 ++++++++++ .../cache/WalModeChangeAbstractSelfTest.java | 45 ++++++++++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java index a01e813cbcc4c..d17f20f221666 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java @@ -122,6 +122,9 @@ public class WalStateManager extends GridCacheSharedManagerAdapter { /** */ private volatile WALDisableContext walDisableContext; + /** Denies or allows WAL disabling. */ + private volatile boolean prohibitDisabling; + /** * Constructor. * @@ -265,6 +268,24 @@ public void onKernalStart() { } } + /** + * Denies or allows WAL disabling with subsequent {@link #init(Collection, boolean)} call. + * + * @param val denial status. + */ + public void prohibitWALDisabling(boolean val) { + prohibitDisabling = val; + } + + /** + * Reports whether WAL disabling with subsequent {@link #init(Collection, boolean)} is denied. + * + * @return denial status. + */ + public boolean prohibitWALDisabling() { + return prohibitDisabling; + } + /** * Initiate WAL mode change operation. * @@ -273,6 +294,9 @@ public void onKernalStart() { * @return Future completed when operation finished. */ public IgniteInternalFuture init(Collection cacheNames, boolean enabled) { + if (!enabled && prohibitDisabling) + return errorFuture("WAL disabling is prohibited."); + if (F.isEmpty(cacheNames)) return errorFuture("Cache names cannot be empty."); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java index 15376411d9afb..8c8ce535e2239 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java @@ -17,17 +17,17 @@ package org.apache.ignite.internal.processors.cache; +import java.util.concurrent.Callable; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.typedef.internal.U; -import java.util.concurrent.Callable; - import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -272,4 +272,45 @@ private void checkEnableDisable(CacheMode mode, CacheAtomicityMode atomicityMode } }); } + + /** + * Test {@link WalStateManager#prohibitWALDisabling(boolean)} feature. + * + * @throws Exception If failed. + */ + public void testDisablingProhibition() throws Exception { + forAllNodes(new IgniteInClosureX() { + @Override public void applyx(Ignite ig) throws IgniteCheckedException { + assert ig instanceof IgniteEx; + + IgniteEx ignite = (IgniteEx)ig; + + createCache(ignite, cacheConfig(CACHE_NAME, PARTITIONED, TRANSACTIONAL)); + + WalStateManager stateMgr = ignite.context().cache().context().walState(); + + assertFalse(stateMgr.prohibitWALDisabling()); + + stateMgr.prohibitWALDisabling(true); + assertTrue(stateMgr.prohibitWALDisabling()); + + try { + walDisable(ignite, CACHE_NAME); + + fail(); + } + catch (Exception e) { + // No-op. + } + + stateMgr.prohibitWALDisabling(false); + assertFalse(stateMgr.prohibitWALDisabling()); + + createCache(ignite, cacheConfig(CACHE_NAME, PARTITIONED, TRANSACTIONAL)); + + assertWalDisable(ignite, CACHE_NAME, true); + assertWalEnable(ignite, CACHE_NAME, true); + } + }); + } } From 175e564036ea06fd5e282f1ad3ed2a54fd3418f6 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Thu, 22 Nov 2018 14:35:35 +0300 Subject: [PATCH 163/403] IGNITE-10047: MVCC: Fixed MVCC coordinator assignment on several nodes failure. This closes #5353. (cherry picked from commit 5477048082711bba0501c73637b4ebaf5270cda4) --- .../managers/discovery/DiscoCache.java | 15 - .../discovery/GridDiscoveryManager.java | 15 +- .../affinity/AffinityAssignment.java | 6 - .../affinity/GridAffinityAssignment.java | 15 +- .../affinity/GridAffinityAssignmentCache.java | 22 +- .../affinity/GridAffinityProcessor.java | 2 +- .../affinity/GridAffinityUtils.java | 2 +- .../affinity/HistoryAffinityAssignment.java | 19 +- .../cache/CacheAffinitySharedManager.java | 13 +- .../processors/cache/ExchangeContext.java | 43 +- .../cache/GridCacheAffinityManager.java | 5 - .../GridCachePartitionExchangeManager.java | 8 - .../dht/GridDhtTxPrepareFuture.java | 23 -- .../GridDhtPartitionsExchangeFuture.java | 43 +- .../GridDhtPartitionsSingleMessage.java | 68 +--- .../topology/GridClientPartitionTopology.java | 7 - .../topology/GridDhtPartitionTopology.java | 4 - .../GridDhtPartitionTopologyImpl.java | 11 - ...OptimisticSerializableTxPrepareFuture.java | 35 +- .../GridNearOptimisticTxPrepareFuture.java | 39 +- ...dNearOptimisticTxPrepareFutureAdapter.java | 114 ------ .../GridNearPessimisticTxPrepareFuture.java | 78 +--- .../near/GridNearTxPrepareRequest.java | 17 - .../near/GridNearTxPrepareResponse.java | 52 +-- .../mvcc/MvccPreviousCoordinatorQueries.java | 13 +- .../processors/cache/mvcc/MvccProcessor.java | 45 +-- .../cache/mvcc/MvccProcessorImpl.java | 372 +++++++++--------- .../cache/mvcc/MvccQueryTrackerImpl.java | 12 +- .../cache/transactions/IgniteTxManager.java | 4 +- ...cAbstractBasicCoordinatorFailoverTest.java | 184 +++++++++ ...vccPartitionedCoordinatorFailoverTest.java | 35 ++ 31 files changed, 467 insertions(+), 854 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java index 70e2013d1065a..e99e4782ba3c8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java @@ -26,7 +26,6 @@ import org.apache.ignite.cluster.BaselineNode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.tostring.GridToStringInclude; @@ -109,14 +108,10 @@ public class DiscoCache { /** */ private final P1 aliveNodePred; - /** */ - private final MvccCoordinator mvccCrd; - /** * @param topVer Topology version. * @param state Current cluster state. * @param loc Local node. - * @param mvccCrd MVCC coordinator node. * @param rmtNodes Remote nodes. * @param allNodes All nodes. * @param srvNodes Server nodes. @@ -135,7 +130,6 @@ public class DiscoCache { AffinityTopologyVersion topVer, DiscoveryDataClusterState state, ClusterNode loc, - MvccCoordinator mvccCrd, List rmtNodes, List allNodes, List srvNodes, @@ -154,7 +148,6 @@ public class DiscoCache { this.topVer = topVer; this.state = state; this.loc = loc; - this.mvccCrd = mvccCrd; this.rmtNodes = rmtNodes; this.allNodes = allNodes; this.srvNodes = srvNodes; @@ -183,13 +176,6 @@ public class DiscoCache { }; } - /** - * @return Mvcc coordinator node. - */ - @Nullable public MvccCoordinator mvccCoordinator() { - return mvccCrd; - } - /** * @return Topology version. */ @@ -475,7 +461,6 @@ public DiscoCache copy(AffinityTopologyVersion ver, @Nullable DiscoveryDataClust ver, state == null ? this.state : state, loc, - mvccCrd, rmtNodes, allNodes, srvNodes, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index 30d169f0be1cb..52c484415dee0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -85,8 +85,6 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; -import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cluster.BaselineTopology; import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; @@ -109,7 +107,6 @@ import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; import org.apache.ignite.lang.IgniteClosure; @@ -659,8 +656,6 @@ private void onDiscovery0( updateClientNodes(node.id()); } - ctx.coordinators().onDiscoveryEvent(type, topSnapshot, topVer, customMsg); - boolean locJoinEvt = type == EVT_NODE_JOINED && node.id().equals(locNode.id()); ChangeGlobalStateFinishMessage stateFinishMsg = null; @@ -793,6 +788,8 @@ else if (customMsg instanceof ChangeGlobalStateMessage) { discoWrk.discoCache = discoCache; if (!isLocDaemon && !ctx.clientDisconnected()) { + ctx.cache().context().coordinators().onLocalJoin(discoEvt); + ctx.cache().context().exchange().onLocalJoin(discoEvt, discoCache); ctx.authentication().onLocalJoin(); @@ -2340,8 +2337,6 @@ public void reconnect() { Collection topSnapshot) { assert topSnapshot.contains(loc); - MvccCoordinator mvccCrd = ctx.coordinators().assignedCoordinator(); - HashSet alives = U.newHashSet(topSnapshot.size()); HashMap nodeMap = U.newHashMap(topSnapshot.size()); @@ -2443,7 +2438,6 @@ else if (node.version().compareTo(minVer) < 0) topVer, state, loc, - mvccCrd, Collections.unmodifiableList(rmtNodes), Collections.unmodifiableList(allNodes), Collections.unmodifiableList(srvNodes), @@ -2615,8 +2609,8 @@ public void scheduleSegmentCheck() { AffinityTopologyVersion.NONE, ctx.state().clusterState(), node, - locNodeOnlyTop - ), locNodeOnlyTop, + locNodeOnlyTop), + locNodeOnlyTop, null); lastSegChkRes.set(false); @@ -3450,7 +3444,6 @@ public DiscoCache createDiscoCacheOnCacheChange( topVer, discoCache.state(), discoCache.localNode(), - discoCache.mvccCoordinator(), discoCache.remoteNodes(), allNodes, discoCache.serverNodes(), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java index eb06d3e60b873..b603c32a8f6ec 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java @@ -22,7 +22,6 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; /** * Cached affinity calculations. @@ -84,9 +83,4 @@ public interface AffinityAssignment { * @return Backup partitions for specified node ID. */ public Set backupPartitions(UUID nodeId); - - /** - * @return Mvcc coordinator. - */ - public MvccCoordinator mvccCoordinator(); } \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java index fe5103681a999..95cf76fc58144 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.util.typedef.internal.S; /** @@ -41,9 +40,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable /** Topology version. */ private final AffinityTopologyVersion topVer; - /** */ - private final MvccCoordinator mvccCrd; - /** Collection of calculated affinity nodes. */ private List> assignment; @@ -74,7 +70,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable this.topVer = topVer; primary = new HashMap<>(); backup = new HashMap<>(); - mvccCrd = null; } /** @@ -84,8 +79,7 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable */ GridAffinityAssignment(AffinityTopologyVersion topVer, List> assignment, - List> idealAssignment, - MvccCoordinator mvccCrd) { + List> idealAssignment) { assert topVer != null; assert assignment != null; assert idealAssignment != null; @@ -93,7 +87,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable this.topVer = topVer; this.assignment = assignment; this.idealAssignment = idealAssignment.equals(assignment) ? assignment : idealAssignment; - this.mvccCrd = mvccCrd; primary = new HashMap<>(); backup = new HashMap<>(); @@ -112,7 +105,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable idealAssignment = aff.idealAssignment; primary = aff.primary; backup = aff.backup; - mvccCrd = aff.mvccCrd; } /** @@ -275,11 +267,6 @@ private void initPrimaryBackupMaps() { } } - /** {@inheritDoc} */ - @Override public MvccCoordinator mvccCoordinator() { - return mvccCrd; - } - /** {@inheritDoc} */ @Override public int hashCode() { return topVer.hashCode(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java index d98ed5f507091..8510a9006526b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java @@ -43,7 +43,6 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.NodeOrderComparator; import org.apache.ignite.internal.managers.discovery.DiscoCache; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents; import org.apache.ignite.internal.processors.cluster.BaselineTopology; import org.apache.ignite.internal.util.future.GridFutureAdapter; @@ -203,28 +202,11 @@ public int groupId() { * @param affAssignment Affinity assignment for topology version. */ public void initialize(AffinityTopologyVersion topVer, List> affAssignment) { - MvccCoordinator mvccCrd = null; - - if (!locCache) - mvccCrd = ctx.cache().context().coordinators().currentCoordinator(topVer); - - initialize(topVer, affAssignment, mvccCrd); - } - - /** - * Initializes affinity with given topology version and assignment. - * - * @param topVer Topology version. - * @param affAssignment Affinity assignment for topology version. - * @param mvccCrd Mvcc coordinator. - */ - public void initialize(AffinityTopologyVersion topVer, List> affAssignment, MvccCoordinator mvccCrd) { assert topVer.compareTo(lastVersion()) >= 0 : "[topVer = " + topVer + ", last=" + lastVersion() + ']'; assert idealAssignment != null; - assert mvccCrd == null || topVer.compareTo(mvccCrd.topologyVersion()) >= 0 : "[mvccCrd=" + mvccCrd + ", topVer=" + topVer + ']'; - GridAffinityAssignment assignment = new GridAffinityAssignment(topVer, affAssignment, idealAssignment, mvccCrd); + GridAffinityAssignment assignment = new GridAffinityAssignment(topVer, affAssignment, idealAssignment); HistoryAffinityAssignment hAff = affCache.put(topVer, new HistoryAffinityAssignment(assignment)); @@ -788,7 +770,7 @@ public void init(GridAffinityAssignmentCache aff) { AffinityAssignment assign = aff.cachedAffinity(aff.lastVersion()); - initialize(aff.lastVersion(), assign.assignment(), assign.mvccCoordinator()); + initialize(aff.lastVersion(), assign.assignment()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java index 4a0908c6071f6..08333c33e131a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java @@ -423,7 +423,7 @@ private Map> keysToNodes(@Nullable final String c try { GridAffinityAssignment assign = assign0 instanceof GridAffinityAssignment ? (GridAffinityAssignment)assign0 : - new GridAffinityAssignment(topVer, assign0.assignment(), assign0.idealAssignment(), assign0.mvccCoordinator()); + new GridAffinityAssignment(topVer, assign0.assignment(), assign0.idealAssignment()); AffinityInfo info = new AffinityInfo( cctx.config().getAffinity(), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityUtils.java index 15d7e4e437ea7..abd5292799958 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityUtils.java @@ -184,7 +184,7 @@ public AffinityJob() { GridAffinityAssignment assign = assign0 instanceof GridAffinityAssignment ? (GridAffinityAssignment)assign0 : - new GridAffinityAssignment(topVer, assign0.assignment(), assign0.idealAssignment(), assign0.mvccCoordinator()); + new GridAffinityAssignment(topVer, assign0.assignment(), assign0.idealAssignment()); return F.t( affinityMessage(ctx, cctx.config().getAffinity()), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java index a9496486fb965..6bdf7f0f52092 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java @@ -17,16 +17,14 @@ package org.apache.ignite.internal.processors.affinity; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; - import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; /** * @@ -42,9 +40,6 @@ public class HistoryAffinityAssignment implements AffinityAssignment { /** */ private final List> idealAssignment; - /** */ - private final MvccCoordinator mvccCrd; - /** * @param assign Assignment. */ @@ -52,12 +47,6 @@ public class HistoryAffinityAssignment implements AffinityAssignment { this.topVer = assign.topologyVersion(); this.assignment = assign.assignment(); this.idealAssignment = assign.idealAssignment(); - this.mvccCrd = assign.mvccCoordinator(); - } - - /** {@inheritDoc} */ - @Override public MvccCoordinator mvccCoordinator() { - return mvccCrd; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 936dc3eb81889..881f8dbb1e5b8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -61,7 +61,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.GridPartitionStateMap; @@ -480,7 +479,6 @@ void onCacheGroupCreated(CacheGroupContext grp) { grp.topology().updateTopologyVersion(topFut, discoCache, - cctx.coordinators().currentCoordinator(), -1, false); @@ -530,7 +528,6 @@ else if (!fetchFuts.containsKey(grp.groupId())) { assert grp != null; GridDhtAffinityAssignmentResponse res = fetchAffinity(topVer, - cctx.coordinators().currentCoordinator(), null, discoCache, grp.affinity(), @@ -555,7 +552,6 @@ else if (!fetchFuts.containsKey(grp.groupId())) { grp.topology().updateTopologyVersion(topFut, discoCache, - cctx.coordinators().currentCoordinator(), -1, false); @@ -1409,7 +1405,6 @@ private void initAffinity(CacheGroupDescriptor desc, fetchFut.init(false); fetchAffinity(evts.topologyVersion(), - cctx.coordinators().currentCoordinator(), evts, evts.discoveryCache(), aff, @@ -1852,7 +1847,6 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign int grpId = fetchFut.groupId(); fetchAffinity(topVer, - cctx.coordinators().currentCoordinator(), fut.events(), fut.events().discoveryCache(), cctx.cache().cacheGroup(grpId).affinity(), @@ -1864,7 +1858,6 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign /** * @param topVer Topology version. - * @param mvccCrd Mvcc coordinator to set in affinity. * @param events Discovery events. * @param discoCache Discovery data cache. * @param affCache Affinity. @@ -1874,7 +1867,6 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign */ private GridDhtAffinityAssignmentResponse fetchAffinity( AffinityTopologyVersion topVer, - MvccCoordinator mvccCrd, @Nullable ExchangeDiscoveryEvents events, DiscoCache discoCache, GridAffinityAssignmentCache affCache, @@ -1887,7 +1879,7 @@ private GridDhtAffinityAssignmentResponse fetchAffinity( if (res == null) { List> aff = affCache.calculate(topVer, events, discoCache); - affCache.initialize(topVer, aff, mvccCrd); + affCache.initialize(topVer, aff); } else { List> idealAff = res.idealAffinityAssignment(discoCache); @@ -1904,7 +1896,7 @@ private GridDhtAffinityAssignmentResponse fetchAffinity( assert aff != null : res; - affCache.initialize(topVer, aff, mvccCrd); + affCache.initialize(topVer, aff); } return res; @@ -2027,7 +2019,6 @@ public IgniteInternalFuture initCoordinatorCaches( @Override public void applyx(IgniteInternalFuture fetchFut) throws IgniteCheckedException { fetchAffinity(prev.topologyVersion(), - null, // Pass null mvcc coordinator, this affinity version should be used for queries. prev.events(), prev.events().discoveryCache(), aff, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeContext.java index 34ed048542c16..4046c98b29414 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeContext.java @@ -17,15 +17,11 @@ package org.apache.ignite.internal.processors.cache; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; -import java.util.UUID; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; -import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.typedef.internal.S; import org.jetbrains.annotations.Nullable; @@ -55,20 +51,11 @@ public class ExchangeContext { /** */ private final boolean compatibilityNode = getBoolean(IGNITE_EXCHANGE_COMPATIBILITY_VER_1, false); - /** */ - private final boolean newMvccCrd; - - /** Currently running mvcc queries, initialized when mvcc coordinator is changed. */ - private Map activeQueries; - /** * @param crd Coordinator flag. - * @param newMvccCrd {@code True} if new coordinator assigned during this exchange. * @param fut Exchange future. */ - public ExchangeContext(boolean crd, boolean newMvccCrd, GridDhtPartitionsExchangeFuture fut) { - this.newMvccCrd = newMvccCrd; - + public ExchangeContext(boolean crd, GridDhtPartitionsExchangeFuture fut) { int protocolVer = exchangeProtocolVersion(fut.firstEventCache().minimumNodeVersion()); if (compatibilityNode || (crd && fut.localJoinExchange())) { @@ -137,34 +124,6 @@ public boolean mergeExchanges() { return merge; } - /** - * @return {@code True} if new node assigned as mvcc coordinator node during this exchange. - */ - public boolean newMvccCoordinator() { - return newMvccCrd; - } - - /** - * @return Active queries. - */ - public Map activeQueries() { - return activeQueries; - } - - /** - * @param nodeId Node ID. - * @param nodeQueries Node queries. - */ - public void addActiveQueries(UUID nodeId, @Nullable GridLongList nodeQueries) { - if (nodeQueries == null) - return; - - if (activeQueries == null) - activeQueries = new HashMap<>(); - - activeQueries.put(nodeId, nodeQueries); - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(ExchangeContext.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java index f811f6db9c927..e20855fef68f9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java @@ -32,7 +32,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityAssignment; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.Nullable; @@ -254,10 +253,6 @@ public AffinityAssignment assignment(AffinityTopologyVersion topVer, AffinityTop return aff0.cachedAffinity(topVer, lastAffChangedTopVer); } - public MvccCoordinator mvccCoordinator(AffinityTopologyVersion topVer) { - return assignment(topVer).mvccCoordinator(); - } - /** * @param key Key to check. * @param topVer Topology version. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 24f505726a7e8..525cb88317395 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -2294,14 +2294,6 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu ClusterNode node = evt.eventNode(); - if ((evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT) && - node.equals(cctx.coordinators().currentCoordinator())) { - if (log.isInfoEnabled()) - log.info("Stop merge, need exchange for mvcc coordinator failure: " + node); - - break; - } - if (!curFut.context().supportsMergeExchanges(node)) { if (log.isInfoEnabled()) log.info("Stop merge, node does not support merge: " + node); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java index d10d329fecf51..6958ac5f7978b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java @@ -905,8 +905,6 @@ private GridNearTxPrepareResponse createPrepareResponse(@Nullable Throwable prep tx.onePhaseCommit(), tx.activeCachesDeploymentEnabled()); - res.mvccSnapshot(tx.mvccSnapshot()); - if (prepErr == null) { if (tx.needReturnValue() || tx.nearOnOriginatingNode() || tx.hasInterceptor()) addDhtValues(res); @@ -1265,27 +1263,6 @@ private void prepare0() { } } - if (req.requestMvccCounter()) { - assert last; - - assert tx.txState().mvccEnabled(); - - try { - // Request snapshot locally only because - // Mvcc Coordinator is expected to be local. - MvccSnapshot snapshot = cctx.coordinators().tryRequestSnapshotLocal(tx); - - assert snapshot != null : tx.topologyVersion(); - - tx.mvccSnapshot(snapshot); - } - catch (ClusterTopologyCheckedException e) { - onDone(e); - - return; - } - } - onEntriesLocked(); // We are holding transaction-level locks for entries here, so we can get next write version. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index b5e49da1ce712..da1a25beb7afe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -91,7 +91,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; @@ -100,7 +99,6 @@ import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; -import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.IgniteInClosureX; @@ -700,25 +698,10 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException { boolean crdNode = crd != null && crd.isLocal(); - MvccCoordinator mvccCrd = firstEvtDiscoCache.mvccCoordinator(); - - boolean mvccCrdChange = mvccCrd != null && - (initialVersion().equals(mvccCrd.topologyVersion()) || activateCluster()); - - // Mvcc coordinator should has been initialized before exchange context is created. - cctx.kernalContext().coordinators().updateCoordinator(mvccCrd); - - exchCtx = new ExchangeContext(crdNode, mvccCrdChange, this); + exchCtx = new ExchangeContext(crdNode, this); cctx.exchange().exchangerBlockingSectionBegin(); - try { - cctx.kernalContext().coordinators().onExchangeStart(mvccCrd, exchCtx, crd); - } - finally { - cctx.exchange().exchangerBlockingSectionEnd(); - } - assert state == null : state; if (crdNode) @@ -728,8 +711,6 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException { if (exchLog.isInfoEnabled()) { exchLog.info("Started exchange init [topVer=" + topVer + - ", mvccCrd=" + mvccCrd + - ", mvccCrdChange=" + mvccCrdChange + ", crd=" + crdNode + ", evt=" + IgniteUtils.gridEventName(firstDiscoEvt.type()) + ", evtNode=" + firstDiscoEvt.eventNode().id() + @@ -819,7 +800,7 @@ else if (msg instanceof WalStateAbstractMessage) cctx.cache().registrateProxyRestart(resolveCacheRequests(exchActions), afterLsnrCompleteFut); - updateTopologies(crdNode, cctx.coordinators().currentCoordinator()); + updateTopologies(crdNode); switch (exchange) { case ALL: { @@ -979,10 +960,9 @@ private void initTopologies() throws IgniteCheckedException { * Updates topology versions and discovery caches on all topologies. * * @param crd Coordinator flag. - * @param mvccCrd Mvcc coordinator. * @throws IgniteCheckedException If failed. */ - private void updateTopologies(boolean crd, MvccCoordinator mvccCrd) throws IgniteCheckedException { + private void updateTopologies(boolean crd) throws IgniteCheckedException { for (CacheGroupContext grp : cctx.cache().cacheGroups()) { if (grp.isLocal()) continue; @@ -1019,7 +999,6 @@ private void updateTopologies(boolean crd, MvccCoordinator mvccCrd) throws Ignit top.updateTopologyVersion( this, events().discoveryCache(), - mvccCrd, updSeq, cacheGroupStopping(grp.groupId())); } @@ -1034,7 +1013,6 @@ private void updateTopologies(boolean crd, MvccCoordinator mvccCrd) throws Ignit for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) { top.updateTopologyVersion(this, events().discoveryCache(), - mvccCrd, -1, cacheGroupStopping(top.groupId())); } @@ -1842,12 +1820,6 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException msg.partitionHistoryCounters(partHistReserved0); } - if (exchCtx.newMvccCoordinator() && cctx.coordinators().currentCoordinatorId().equals(node.id())) { - Map activeQueries = exchCtx.activeQueries(); - - msg.activeQueries(activeQueries != null ? activeQueries.get(cctx.localNodeId()) : null); - } - if ((stateChangeExchange() || dynamicCacheStartExchange()) && exchangeLocE != null) msg.setError(exchangeLocE); else if (localJoinExchange()) @@ -2101,8 +2073,7 @@ public void finishMerged() { tryToPerformLocalSnapshotOperation(); if (err == null) - cctx.coordinators().onExchangeDone(exchCtx.newMvccCoordinator(), exchCtx.events().discoveryCache(), - exchCtx.activeQueries()); + cctx.coordinators().onExchangeDone(exchCtx.events().discoveryCache()); // Create and destory caches and cache proxies. cctx.cache().onExchangeDone(initialVersion(), exchActions, err); @@ -2677,9 +2648,6 @@ public void waitAndReplyToNode(final UUID nodeId, final GridDhtPartitionsSingleM */ private void processSingleMessage(UUID nodeId, GridDhtPartitionsSingleMessage msg) { if (msg.client()) { - if (msg.activeQueries() != null) - cctx.coordinators().processClientActiveQueries(nodeId, msg.activeQueries()); - waitAndReplyToNode(nodeId, msg); return; @@ -3239,9 +3207,6 @@ private void finishExchangeOnCoordinator(@Nullable Collection sndRe for (Map.Entry e : msgs.entrySet()) { GridDhtPartitionsSingleMessage msg = e.getValue(); - if (exchCtx.newMvccCoordinator()) - exchCtx.addActiveQueries(e.getKey(), msg.activeQueries()); - // Apply update counters after all single messages are received. for (Map.Entry entry : msg.partitions().entrySet()) { Integer grpId = entry.getKey(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index b5de155255cc4..8db717651b298 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -30,7 +30,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; -import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; @@ -109,9 +108,6 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes */ private GridDhtPartitionsFullMessage finishMsg; - /** */ - private GridLongList activeQryTrackers; - /** * Required by {@link Externalizable}. */ @@ -135,20 +131,6 @@ public GridDhtPartitionsSingleMessage(GridDhtPartitionExchangeId exchId, this.compress = compress; } - /** - * @return Active queries started with previous coordinator. - */ - GridLongList activeQueries() { - return activeQryTrackers; - } - - /** - * @param activeQrys Active queries started with previous coordinator. - */ - void activeQueries(GridLongList activeQrys) { - this.activeQryTrackers = activeQrys; - } - /** * @param finishMsg Exchange finish message (used to restore exchange state on new coordinator). */ @@ -474,65 +456,58 @@ public void setError(Exception ex) { switch (writer.state()) { case 6: - if (!writer.writeMessage("activeQryTrackers", activeQryTrackers)) - return false; - - writer.incrementState(); - - case 7: if (!writer.writeBoolean("client", client)) return false; writer.incrementState(); - case 8: + case 7: if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 9: + case 8: if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); - case 10: + case 9: if (!writer.writeMessage("finishMsg", finishMsg)) return false; writer.incrementState(); - case 11: + case 10: if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT)) return false; writer.incrementState(); - case 12: + case 11: if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes)) return false; writer.incrementState(); - case 13: + case 12: if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes)) return false; writer.incrementState(); - case 14: + case 13: if (!writer.writeByteArray("partsBytes", partsBytes)) return false; writer.incrementState(); - case 15: + case 14: if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes)) return false; writer.incrementState(); - } return true; @@ -550,14 +525,6 @@ public void setError(Exception ex) { switch (reader.state()) { case 6: - activeQryTrackers = reader.readMessage("activeQryTrackers"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 7: client = reader.readBoolean("client"); if (!reader.isLastRead()) @@ -565,7 +532,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 8: + case 7: dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false); if (!reader.isLastRead()) @@ -573,7 +540,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 9: + case 8: errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) @@ -581,7 +548,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 10: + case 9: finishMsg = reader.readMessage("finishMsg"); if (!reader.isLastRead()) @@ -589,7 +556,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 11: + case 10: grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT); if (!reader.isLastRead()) @@ -597,7 +564,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 12: + case 11: partCntrsBytes = reader.readByteArray("partCntrsBytes"); if (!reader.isLastRead()) @@ -605,7 +572,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 13: + case 12: partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes"); if (!reader.isLastRead()) @@ -613,7 +580,7 @@ public void setError(Exception ex) { reader.incrementState(); - case 14: + case 13: partsBytes = reader.readByteArray("partsBytes"); if (!reader.isLastRead()) @@ -621,14 +588,13 @@ public void setError(Exception ex) { reader.incrementState(); - case 15: + case 14: partsSizesBytes = reader.readByteArray("partsSizesBytes"); if (!reader.isLastRead()) return false; reader.incrementState(); - } return reader.afterMessageRead(GridDhtPartitionsSingleMessage.class); @@ -641,7 +607,7 @@ public void setError(Exception ex) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 16; + return 15; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java index 3c4eba15337d7..01908af8c9d4d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java @@ -48,7 +48,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.util.F0; import org.apache.ignite.internal.util.GridAtomicLong; import org.apache.ignite.internal.util.GridPartitionStateMap; @@ -201,11 +200,6 @@ private String mapString(GridDhtPartitionMap map) { lock.readLock().unlock(); } - /** {@inheritDoc} */ - @Override public MvccCoordinator mvccCoordinator() { - throw new UnsupportedOperationException(); - } - /** {@inheritDoc} */ @Override public boolean holdsLock() { return lock.isWriteLockedByCurrentThread() || lock.getReadHoldCount() > 0; @@ -215,7 +209,6 @@ private String mapString(GridDhtPartitionMap map) { @Override public void updateTopologyVersion( GridDhtTopologyFuture exchFut, DiscoCache discoCache, - MvccCoordinator mvccCrd, long updSeq, boolean stopping ) throws IgniteInterruptedCheckedException { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index 2f5e57bace9dc..81eab849a3561 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -38,7 +38,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.jetbrains.annotations.Nullable; @@ -78,7 +77,6 @@ public interface GridDhtPartitionTopology { public void updateTopologyVersion( GridDhtTopologyFuture exchFut, DiscoCache discoCache, - MvccCoordinator mvccCrd, long updateSeq, boolean stopping ) throws IgniteInterruptedCheckedException; @@ -432,6 +430,4 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, * @param updateRebalanceVer {@code True} if need check rebalance state. */ public void onExchangeDone(GridDhtPartitionsExchangeFuture fut, AffinityAssignment assignment, boolean updateRebalanceVer); - - public MvccCoordinator mvccCoordinator(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index c32a1de2ef9bf..cfb5a961d1789 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -56,7 +56,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.util.F0; import org.apache.ignite.internal.util.GridAtomicLong; import org.apache.ignite.internal.util.GridLongList; @@ -152,9 +151,6 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology { /** */ private volatile AffinityTopologyVersion rebalancedTopVer = AffinityTopologyVersion.NONE; - /** */ - private volatile MvccCoordinator mvccCrd; - /** * @param ctx Cache shared context. * @param grp Cache group. @@ -246,11 +242,6 @@ private String mapString(GridDhtPartitionMap map) { lock.readLock().unlock(); } - /** {@inheritDoc} */ - @Override public MvccCoordinator mvccCoordinator() { - return mvccCrd; - } - /** {@inheritDoc} */ @Override public boolean holdsLock() { return lock.isWriteLockedByCurrentThread() || lock.getReadHoldCount() > 0; @@ -260,7 +251,6 @@ private String mapString(GridDhtPartitionMap map) { @Override public void updateTopologyVersion( GridDhtTopologyFuture exchFut, @NotNull DiscoCache discoCache, - MvccCoordinator mvccCrd, long updSeq, boolean stopping ) throws IgniteInterruptedCheckedException { @@ -287,7 +277,6 @@ private String mapString(GridDhtPartitionMap map) { lastTopChangeVer = exchTopVer; this.discoCache = discoCache; - this.mvccCrd = mvccCrd; } finally { lock.writeLock().unlock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java index 2c8fe219cb52f..30e6953baf233 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java @@ -38,7 +38,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; @@ -59,7 +58,6 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM; -import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.noCoordinatorError; import static org.apache.ignite.transactions.TransactionState.PREPARED; import static org.apache.ignite.transactions.TransactionState.PREPARING; @@ -196,9 +194,6 @@ private void prepareError(Throwable e) { if (keyLockFut != null) keyLockFut.onDone(e); - - if (mvccVerFut != null) - mvccVerFut.onDone(); } /** {@inheritDoc} */ @@ -349,25 +344,11 @@ private void prepare( boolean hasNearCache = false; - MvccCoordinator mvccCrd = null; - for (IgniteTxEntry write : writes) { map(write, topVer, mappings, txMapping, remap, topLocked); - GridCacheContext cctx = write.context(); - - if (cctx.isNear()) + if (write.context().isNear()) hasNearCache = true; - - if (cctx.mvccEnabled() && mvccCrd == null) { - mvccCrd = cctx.affinity().mvccCoordinator(topVer); - - if (mvccCrd == null) { - onDone(noCoordinatorError(topVer)); - - return; - } - } } for (IgniteTxEntry read : reads) @@ -383,8 +364,6 @@ private void prepare( return; } - assert !tx.txState().mvccEnabled() || mvccCrd != null || F.isEmpty(writes); - tx.addEntryMapping(mappings.values()); cctx.mvcc().recheckPendingLocks(); @@ -396,16 +375,12 @@ private void prepare( MiniFuture locNearEntriesFut = null; - int lockCnt = keyLockFut != null ? 1 : 0; - // Create futures in advance to have all futures when process {@link GridNearTxPrepareResponse#clientRemapVersion}. for (GridDistributedTxMapping m : mappings.values()) { assert !m.empty(); MiniFuture fut = new MiniFuture(this, m, ++miniId); - lockCnt++; - add((IgniteInternalFuture)fut); if (m.primary().isLocal() && m.hasNearCacheEntries() && m.hasColocatedCacheEntries()) { @@ -414,14 +389,9 @@ private void prepare( locNearEntriesFut = fut; add((IgniteInternalFuture)new MiniFuture(this, m, ++miniId)); - - lockCnt++; } } - if (mvccCrd != null) - initMvccVersionFuture(lockCnt, remap); - Collection> futs = (Collection)futures(); Iterator> it = futs.iterator(); @@ -986,9 +956,6 @@ void onResult(final GridNearTxPrepareResponse res, boolean updateMapping) { // Finish this mini future (need result only on client node). onDone(parent.cctx.kernalContext().clientNode() ? res : null); - - if (parent.mvccVerFut != null) - parent.mvccVerFut.onLockReceived(); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java index e1bfa90b69a02..5883b941148d8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java @@ -43,7 +43,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; @@ -69,7 +68,6 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM; -import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.noCoordinatorError; import static org.apache.ignite.transactions.TransactionState.PREPARED; import static org.apache.ignite.transactions.TransactionState.PREPARING; @@ -382,18 +380,6 @@ else if (write.context().isColocated()) tx.colocatedLocallyMapped(true); } - if (write.context().mvccEnabled()) { - MvccCoordinator mvccCrd = write.context().affinity().mvccCoordinator(topVer); - - if (mvccCrd == null) { - onDone(noCoordinatorError(topVer)); - - return; - } - - initMvccVersionFuture(keyLockFut != null ? 2 : 1, remap); - } - if (keyLockFut != null) keyLockFut.onAllKeysAdded(); @@ -438,8 +424,6 @@ private void prepare( boolean hasNearCache = false; - MvccCoordinator mvccCrd = null; - for (IgniteTxEntry write : writes) { write.clearEntryReadVersion(); @@ -449,16 +433,6 @@ private void prepare( // an exception occurred while transaction mapping, stop further processing break; - if (write.context().mvccEnabled() && mvccCrd == null) { - mvccCrd = write.context().affinity().mvccCoordinator(topVer); - - if (mvccCrd == null) { - onDone(noCoordinatorError(topVer)); - - break; - } - } - if (write.context().isNear()) hasNearCache = true; @@ -498,11 +472,6 @@ else if (write.context().isColocated()) return; } - assert !tx.txState().mvccEnabled() || mvccCrd != null; - - if (mvccCrd != null) - initMvccVersionFuture(keyLockFut != null ? 2 : 1, remap); - if (keyLockFut != null) keyLockFut.onAllKeysAdded(); @@ -526,12 +495,8 @@ else if (write.context().isColocated()) private void proceedPrepare(final Queue mappings) { final GridDistributedTxMapping m = mappings.poll(); - if (m == null) { - if (mvccVerFut != null) - mvccVerFut.onLockReceived(); - + if (m == null) return; - } proceedPrepare(m, mappings); } @@ -1039,8 +1004,6 @@ void onResult(final GridNearTxPrepareResponse res) { // Proceed prepare before finishing mini future. if (mappings != null) parent.proceedPrepare(mappings); - else if (parent.mvccVerFut != null) - parent.mvccVerFut.onLockReceived(); // Finish this mini future. onDone((GridNearTxPrepareResponse)null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java index 6f541d330b5a1..913e9bc4c92eb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java @@ -18,15 +18,11 @@ package org.apache.ignite.internal.processors.cache.distributed.near; import java.util.Collection; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; -import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; -import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshotResponseListener; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; @@ -35,7 +31,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.lang.IgniteInClosure; import org.jetbrains.annotations.Nullable; /** @@ -45,18 +40,10 @@ public abstract class GridNearOptimisticTxPrepareFutureAdapter extends GridNearT /** */ private static final long serialVersionUID = 7460376140787916619L; - /** */ - private static final AtomicIntegerFieldUpdater LOCK_CNT_UPD = - AtomicIntegerFieldUpdater.newUpdater(MvccSnapshotFutureExt.class, "lockCnt"); - /** */ @GridToStringExclude protected KeyLockFuture keyLockFut; - /** */ - @GridToStringExclude - protected MvccSnapshotFutureExt mvccVerFut; - /** * @param cctx Context. * @param tx Transaction. @@ -213,29 +200,6 @@ protected final void prepareOnTopology(final boolean remap, @Nullable final Runn */ protected abstract void prepare0(boolean remap, boolean topLocked); - /** - * @param lockCnt Expected number of lock responses. - * @param remap Remap flag. - */ - @SuppressWarnings("unchecked") - final void initMvccVersionFuture(int lockCnt, boolean remap) { - if (!remap) { - mvccVerFut = new MvccSnapshotFutureExt(); - - mvccVerFut.init(lockCnt); - - if (keyLockFut != null) - keyLockFut.listen(mvccVerFut); - - add((IgniteInternalFuture)mvccVerFut); - } - else { - assert mvccVerFut != null; - - mvccVerFut.init(lockCnt); - } - } - /** * @param e Exception. * @param timedOut {@code True} if timed out. @@ -314,82 +278,4 @@ private void checkLocks() { return S.toString(KeyLockFuture.class, this, super.toString()); } } - - /** - * - */ - class MvccSnapshotFutureExt extends GridFutureAdapter implements MvccSnapshotResponseListener, IgniteInClosure> { - /** */ - private static final long serialVersionUID = 5883078648683911226L; - - /** */ - volatile int lockCnt; - - /** {@inheritDoc} */ - @Override public void apply(IgniteInternalFuture keyLockFut) { - try { - keyLockFut.get(); - - onLockReceived(); - } - catch (IgniteCheckedException e) { - if (log.isDebugEnabled()) - log.debug("MvccSnapshotFutureExt ignores key lock future failure: " + e); - } - } - - /** - * @param lockCnt Expected number of lock responses. - */ - void init(int lockCnt) { - assert lockCnt > 0; - - this.lockCnt = lockCnt; - - assert !isDone(); - } - - /** */ - void onLockReceived() { - int remaining = LOCK_CNT_UPD.decrementAndGet(this); - - assert remaining >= 0 : remaining; - - if (remaining == 0) { - try { - MvccSnapshot snapshot = cctx.coordinators().tryRequestSnapshotLocal(tx); - - if (snapshot != null) - onResponse(snapshot); - else - cctx.coordinators().requestSnapshotAsync(tx, this); - } - catch (ClusterTopologyCheckedException e) { - onError(e); - } - } - } - - /** {@inheritDoc} */ - @Override public void onResponse(MvccSnapshot res) { - tx.mvccSnapshot(res); - - onDone(); - } - - /** {@inheritDoc} */ - @Override public void onError(IgniteCheckedException e) { - if (e instanceof ClusterTopologyCheckedException) - ((ClusterTopologyCheckedException)e).retryReadyFuture(cctx.nextAffinityReadyFuture(tx.topologyVersion())); - - ERR_UPD.compareAndSet(GridNearOptimisticTxPrepareFutureAdapter.this, null, e); - - onDone(); - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(MvccSnapshotFutureExt.class, this, super.toString()); - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java index beff162fb444c..22f7e56551b9d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java @@ -34,11 +34,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator; -import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; -import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshotResponseListener; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException; @@ -52,7 +49,6 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM; -import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.noCoordinatorError; import static org.apache.ignite.transactions.TransactionState.PREPARED; import static org.apache.ignite.transactions.TransactionState.PREPARING; @@ -285,8 +281,6 @@ private void preparePessimistic() { AffinityTopologyVersion topVer = tx.topologyVersion(); - MvccCoordinator mvccCrd = null; - boolean hasNearCache = false; Map> txNodes; @@ -324,16 +318,6 @@ private void preparePessimistic() { else nodes = cacheCtx.affinity().nodesByKey(txEntry.key(), topVer); - if (tx.mvccSnapshot() == null && mvccCrd == null && cacheCtx.mvccEnabled()) { - mvccCrd = cacheCtx.affinity().mvccCoordinator(topVer); - - if (mvccCrd == null) { - onDone(noCoordinatorError(topVer)); - - return; - } - } - if (F.isEmpty(nodes)) { onDone(new ClusterTopologyServerNotFoundException("Failed to map keys to nodes (partition " + "is not mapped to any node) [key=" + txEntry.key() + @@ -359,8 +343,6 @@ private void preparePessimistic() { txNodes = txMapping.transactionNodes(); } - assert !tx.txState().mvccEnabled() || tx.mvccSnapshot() != null || mvccCrd != null; - tx.transactionNodes(txNodes); if (!hasNearCache) @@ -379,16 +361,6 @@ private void preparePessimistic() { for (final GridDistributedTxMapping m : mappings.values()) { final ClusterNode primary = m.primary(); - boolean needCntr = false; - - if (mvccCrd != null) { - if (tx.onePhaseCommit() || mvccCrd.nodeId().equals(primary.id())) { - needCntr = true; - - mvccCrd = null; - } - } - if (primary.isLocal()) { if (m.hasNearCacheEntries() && m.hasColocatedCacheEntries()) { GridNearTxPrepareRequest nearReq = createRequest(txNodes, @@ -397,8 +369,6 @@ private void preparePessimistic() { m.nearEntriesReads(), m.nearEntriesWrites()); - nearReq.requestMvccCounter(needCntr); - prepareLocal(nearReq, m, ++miniId, true); GridNearTxPrepareRequest colocatedReq = createRequest(txNodes, @@ -412,8 +382,6 @@ private void preparePessimistic() { else { GridNearTxPrepareRequest req = createRequest(txNodes, m, timeout, m.reads(), m.writes()); - req.requestMvccCounter(needCntr); - prepareLocal(req, m, ++miniId, m.hasNearCacheEntries()); } } @@ -424,8 +392,6 @@ private void preparePessimistic() { m.reads(), m.writes()); - req.requestMvccCounter(needCntr); - final MiniFuture fut = new MiniFuture(m, ++miniId); req.miniId(fut.futureId()); @@ -458,16 +424,6 @@ private void preparePessimistic() { } } - if (mvccCrd != null) { - assert !tx.onePhaseCommit(); - - MvccSnapshotFutureExt fut = new MvccSnapshotFutureExt(); - - cctx.coordinators().requestSnapshotAsync(tx, fut); - - add((IgniteInternalFuture)fut); - } - markInitialized(); } @@ -515,35 +471,6 @@ private void preparePessimistic() { "super", super.toString()); } - /** - * - */ - private class MvccSnapshotFutureExt extends GridFutureAdapter implements MvccSnapshotResponseListener { - /** {@inheritDoc} */ - @Override public void onResponse(MvccSnapshot res) { - tx.mvccSnapshot(res); - - onDone(); - } - - /** {@inheritDoc} */ - @Override public void onError(IgniteCheckedException e) { - if (log.isDebugEnabled()) - log.debug("Error on tx prepare [fut=" + this + ", err=" + e + ", tx=" + tx + ']'); - - if (ERR_UPD.compareAndSet(GridNearPessimisticTxPrepareFuture.this, null, e)) - tx.setRollbackOnly(); - - onDone(e); - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(MvccSnapshotFutureExt.class, this, super.toString()); - } - } - - /** */ private class MiniFuture extends GridFutureAdapter { /** */ @@ -583,9 +510,6 @@ void onResult(GridNearTxPrepareResponse res, boolean updateMapping) { if (res.error() != null) onError(res.error()); else { - if (res.mvccSnapshot() != null) - tx.mvccSnapshot(res.mvccSnapshot()); - onPrepareResponse(m, res, updateMapping); onDone(res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java index cb17e6a9e7373..7b2d26f64df62 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java @@ -58,9 +58,6 @@ public class GridNearTxPrepareRequest extends GridDistributedTxPrepareRequest { /** */ private static final int ALLOW_WAIT_TOP_FUT_FLAG_MASK = 0x10; - /** */ - private static final int REQUEST_MVCC_CNTR_FLAG_MASK = 0x20; - /** Future ID. */ private IgniteUuid futId; @@ -152,20 +149,6 @@ public GridNearTxPrepareRequest( setFlag(allowWaitTopFut, ALLOW_WAIT_TOP_FUT_FLAG_MASK); } - /** - * @return {@code True} if need request MVCC counter on primary node on prepare step. - */ - public boolean requestMvccCounter() { - return isFlag(REQUEST_MVCC_CNTR_FLAG_MASK); - } - - /** - * @param val {@code True} if need request MVCC counter on primary node on prepare step. - */ - public void requestMvccCounter(boolean val) { - setFlag(val, REQUEST_MVCC_CNTR_FLAG_MASK); - } - /** * @return {@code True} if it is safe for first client request to wait for topology future * completion. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java index 67f7116b7a909..ef547ccaed0a6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java @@ -33,7 +33,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheReturn; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxPrepareResponse; -import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -98,9 +97,6 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse /** Not {@code null} if client node should remap transaction. */ private AffinityTopologyVersion clientRemapVer; - /** */ - private MvccSnapshot mvccSnapshot; - /** * Empty constructor required by {@link Externalizable}. */ @@ -149,20 +145,6 @@ public GridNearTxPrepareResponse( flags |= NEAR_PREPARE_ONE_PHASE_COMMIT_FLAG_MASK; } - /** - * @param mvccSnapshot Mvcc info. - */ - public void mvccSnapshot(MvccSnapshot mvccSnapshot) { - this.mvccSnapshot = mvccSnapshot; - } - - /** - * @return Mvcc info. - */ - @Nullable public MvccSnapshot mvccSnapshot() { - return mvccSnapshot; - } - /** * @return One-phase commit state on primary node. */ @@ -407,41 +389,34 @@ public boolean hasOwnedValue(IgniteTxKey key) { writer.incrementState(); case 16: - if (!writer.writeMessage("mvccSnapshot", mvccSnapshot)) - return false; - - writer.incrementState(); - - case 17: if (!writer.writeCollection("ownedValKeys", ownedValKeys, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 18: + case 17: if (!writer.writeCollection("ownedValVals", ownedValVals, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 19: + case 18: if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG)) return false; writer.incrementState(); - case 20: + case 19: if (!writer.writeMessage("retVal", retVal)) return false; writer.incrementState(); - case 21: + case 20: if (!writer.writeMessage("writeVer", writeVer)) return false; writer.incrementState(); - } return true; @@ -499,14 +474,6 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); case 16: - mvccSnapshot = reader.readMessage("mvccSnapshot"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 17: ownedValKeys = reader.readCollection("ownedValKeys", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -514,7 +481,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); - case 18: + case 17: ownedValVals = reader.readCollection("ownedValVals", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -522,7 +489,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); - case 19: + case 18: pending = reader.readCollection("pending", MessageCollectionItemType.MSG); if (!reader.isLastRead()) @@ -530,7 +497,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); - case 20: + case 19: retVal = reader.readMessage("retVal"); if (!reader.isLastRead()) @@ -538,14 +505,13 @@ public boolean hasOwnedValue(IgniteTxKey key) { reader.incrementState(); - case 21: + case 20: writeVer = reader.readMessage("writeVer"); if (!reader.isLastRead()) return false; reader.incrementState(); - } return reader.afterMessageRead(GridNearTxPrepareResponse.class); @@ -558,7 +524,7 @@ public boolean hasOwnedValue(IgniteTxKey key) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 22; + return 21; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccPreviousCoordinatorQueries.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccPreviousCoordinatorQueries.java index cd7560f40a205..6218bc026fa50 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccPreviousCoordinatorQueries.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccPreviousCoordinatorQueries.java @@ -19,7 +19,6 @@ import java.util.Collection; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -58,7 +57,7 @@ class MvccPreviousCoordinatorQueries { * @param nodes Cluster nodes. * @param mgr Discovery manager. */ - void init(Map nodeQueries, Collection nodes, GridDiscoveryManager mgr) { + void init(GridLongList nodeQueries, Collection nodes, GridDiscoveryManager mgr) { synchronized (this) { assert !initDone; assert waitNodes == null; @@ -66,18 +65,14 @@ void init(Map nodeQueries, Collection nodes, Gr waitNodes = new HashSet<>(); for (ClusterNode node : nodes) { - if ((nodeQueries == null || !nodeQueries.containsKey(node.id())) && - mgr.alive(node) && - !F.contains(rcvd, node.id())) + if (!node.isLocal() && mgr.alive(node) && !F.contains(rcvd, node.id())) waitNodes.add(node.id()); } initDone = waitNodes.isEmpty(); - if (nodeQueries != null) { - for (Map.Entry e : nodeQueries.entrySet()) - mergeToActiveQueries(e.getKey(), e.getValue()); - } + if (nodeQueries != null) + mergeToActiveQueries(mgr.localNode().id(), nodeQueries); if (initDone && !prevQueriesDone) prevQueriesDone = activeQueries.isEmpty() && rcvdAcks.isEmpty(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessor.java index a926acf032735..fd45c7a053585 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessor.java @@ -17,21 +17,16 @@ package org.apache.ignite.internal.processors.cache.mvcc; -import java.util.Collection; -import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; -import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.internal.IgniteDiagnosticPrepareContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.managers.discovery.DiscoCache; -import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.processors.GridProcessor; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.ExchangeContext; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.util.GridLongList; @@ -42,31 +37,18 @@ */ public interface MvccProcessor extends GridProcessor { /** - * @param evtType Event type. - * @param nodes Current nodes. - * @param topVer Topology version. - * @param customMsg Message - */ - void onDiscoveryEvent(int evtType, Collection nodes, long topVer, - @Nullable DiscoveryCustomMessage customMsg); - - /** - * Exchange start callback. + * Local join callback. * - * @param mvccCrd Mvcc coordinator. - * @param exchCtx Exchange context. - * @param exchCrd Exchange coordinator. + * @param evt Discovery event. */ - void onExchangeStart(MvccCoordinator mvccCrd, ExchangeContext exchCtx, ClusterNode exchCrd); + void onLocalJoin(DiscoveryEvent evt); /** * Exchange done callback. * - * @param newCoord New coordinator flag. * @param discoCache Disco cache. - * @param activeQueries Active queries. */ - void onExchangeDone(boolean newCoord, DiscoCache discoCache, Map activeQueries); + void onExchangeDone(DiscoCache discoCache); /** * @param nodeId Node ID @@ -74,33 +56,16 @@ void onDiscoveryEvent(int evtType, Collection nodes, long topVer, */ void processClientActiveQueries(UUID nodeId, @Nullable GridLongList activeQueries); - /** - * @return Mvcc coordinator received from discovery event. - */ - @Nullable MvccCoordinator assignedCoordinator(); - /** * @return Coordinator. */ @Nullable MvccCoordinator currentCoordinator(); - /** - * Check that the given topology is greater or equals to coordinator's one and returns current coordinator. - * @param topVer Topology version. - * @return Mvcc coordinator. - */ - @Nullable MvccCoordinator currentCoordinator(AffinityTopologyVersion topVer); - /** * @return Current coordinator node ID. */ UUID currentCoordinatorId(); - /** - * @param curCrd Coordinator. - */ - void updateCoordinator(MvccCoordinator curCrd); - /** * @param crdVer Mvcc coordinator version. * @param cntr Mvcc counter. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java index 9c8254b9b0618..e8f3c5386624e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java @@ -48,15 +48,14 @@ import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.managers.communication.GridMessageListener; +import org.apache.ignite.internal.managers.discovery.CustomEventListener; import org.apache.ignite.internal.managers.discovery.DiscoCache; -import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch; import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest; -import org.apache.ignite.internal.processors.cache.ExchangeContext; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.KeyCacheObject; @@ -108,13 +107,10 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; -import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; +import static org.apache.ignite.events.EventType.EVT_NODE_JOINED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; -import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; -import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE_COORDINATOR; -import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; import static org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker.MVCC_TRACKER_ID_NA; import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.MVCC_COUNTER_NA; @@ -159,14 +155,11 @@ static void coordinatorAssignClosure(IgniteClosure, Clus } /** Topology version when local node was assigned as coordinator. */ - private long crdVer; + private volatile long crdVer; /** */ private volatile MvccCoordinator curCrd; - /** */ - private volatile MvccCoordinator assignedCrd; - /** */ private TxLog txLog; @@ -226,6 +219,9 @@ static void coordinatorAssignClosure(IgniteClosure, Clus /** Flag whether all nodes in cluster support MVCC. */ private volatile boolean mvccSupported = true; + /** Flag whether coordinator was changed by the last discovery event. */ + private volatile boolean crdChanged; + /** * Maps failed node id to votes accumulator for that node. */ @@ -242,10 +238,21 @@ public MvccProcessorImpl(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void start() throws IgniteCheckedException { - ctx.event().addLocalEventListener(new CacheCoordinatorNodeFailListener(), - EVT_NODE_FAILED, EVT_NODE_LEFT); + ctx.event().addLocalEventListener(new GridLocalEventListener() { + @Override public void onEvent(Event evt) { + onDiscovery((DiscoveryEvent)evt); + } + }, + EVT_NODE_FAILED, EVT_NODE_LEFT, EVT_NODE_JOINED); ctx.io().addMessageListener(TOPIC_CACHE_COORDINATOR, new CoordinatorMessageListener()); + + ctx.discovery().setCustomEventListener(DynamicCacheChangeBatch.class, + new CustomEventListener() { + @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, DynamicCacheChangeBatch msg) { + checkMvccCacheStarted(msg); + } + }); } /** {@inheritDoc} */ @@ -352,72 +359,171 @@ private void txLogPageStoreInit(IgniteCacheDatabaseSharedManager mgr) throws Ign } /** {@inheritDoc} */ - @Override public void onDiscoveryEvent(int evtType, Collection nodes, long topVer, - @Nullable DiscoveryCustomMessage customMsg) { - if (evtType == EVT_NODE_METRICS_UPDATED) - return; + @Override public void onExchangeDone(DiscoCache discoCache) { + MvccCoordinator curCrd0 = curCrd; - if (evtType == EVT_DISCOVERY_CUSTOM_EVT) - checkMvccCacheStarted(customMsg); - else - assignMvccCoordinator(evtType, nodes, topVer); + if (crdChanged) { + // Rollback all transactions with old snapshots. + ctx.cache().context().tm().rollbackMvccTxOnCoordinatorChange(); + + // Complete init future if local node is a new coordinator. All previous txs are already completed here. + if (crdVer != 0 && !initFut.isDone()) { + assert curCrd0 != null && curCrd0.nodeId().equals(ctx.localNodeId()); + + initFut.onDone(); + } + + crdChanged = false; + } + else { + if (curCrd0 != null && ctx.localNodeId().equals(curCrd0.nodeId()) && discoCache != null) + cleanupOrphanedServerTransactions(discoCache.serverNodes()); + } } /** {@inheritDoc} */ - @Override public void onExchangeStart(MvccCoordinator mvccCrd, ExchangeContext exchCtx, ClusterNode exchCrd) { - if (!exchCtx.newMvccCoordinator()) + @Override public void onLocalJoin(DiscoveryEvent evt) { + assert evt.type() == EVT_NODE_JOINED && ctx.localNodeId().equals(evt.eventNode().id()); + + onCoordinatorChanged(evt.topologyNodes(), evt.topologyVersion(), false); + } + + /** + * Discovery listener. Note: initial join event is handled by {@link MvccProcessorImpl#onLocalJoin(DiscoveryEvent)} + * method. + * + * @param evt Discovery event. + */ + private void onDiscovery(DiscoveryEvent evt) { + assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_JOINED; + + UUID nodeId = evt.eventNode().id(); + + MvccCoordinator curCrd0 = curCrd; + + if (evt.type() == EVT_NODE_JOINED) { + if (curCrd0 == null) // Handle join event only if coordinator has not been elected yet. + onCoordinatorChanged(evt.topologyNodes(), evt.topologyVersion(), false); + return; + } - GridLongList activeQryTrackers = collectActiveQueryTrackers(); + // Process mvcc coordinator left event on the rest nodes. + if (nodeId.equals(curCrd0.nodeId())) { + // 1. Notify all listeners waiting for a snapshot. + Map map = snapLsnrs.remove(nodeId); - exchCtx.addActiveQueries(ctx.localNodeId(), activeQryTrackers); + if (map != null) { + ClusterTopologyCheckedException ex = new ClusterTopologyCheckedException("Failed to request mvcc " + + "version, coordinator failed: " + nodeId); - if (exchCrd == null || !mvccCrd.nodeId().equals(exchCrd.id())) { - try { - sendMessage(mvccCrd.nodeId(), new MvccActiveQueriesMessage(activeQryTrackers)); + MvccSnapshotResponseListener lsnr; + + for (Long id : map.keySet()) { + if ((lsnr = map.remove(id)) != null) + lsnr.onError(ex); + } } - catch (IgniteCheckedException e) { - U.error(log, "Failed to send active queries to mvcc coordinator: " + e); + + // 2. Notify acknowledge futures. + for (WaitAckFuture fut : ackFuts.values()) + fut.onNodeLeft(nodeId); + + // 3. Process coordinator change. + onCoordinatorChanged(evt.topologyNodes(), evt.topologyVersion(), true); + } + // Process node left event on the current mvcc coordinator. + else if (curCrd0.nodeId().equals(ctx.localNodeId())) { + // 1. Notify active queries. + activeQueries.onNodeFailed(nodeId); + + // 2. Notify previous queries. + prevCrdQueries.onNodeFailed(nodeId); + + // 3. Recover transactions started by the failed node. + recoveryBallotBoxes.forEach((nearNodeId, ballotBox) -> { + // Put synthetic vote from another failed node + ballotBox.vote(nodeId); + + tryFinishRecoveryVoting(nearNodeId, ballotBox); + }); + + if (evt.eventNode().isClient()) { + RecoveryBallotBox ballotBox = recoveryBallotBoxes + .computeIfAbsent(nodeId, uuid -> new RecoveryBallotBox()); + + ballotBox + .voters(evt.topologyNodes().stream().map(ClusterNode::id).collect(Collectors.toList())); + + tryFinishRecoveryVoting(nodeId, ballotBox); } } } - /** {@inheritDoc} */ - @Override public void onExchangeDone(boolean newCrd, DiscoCache discoCache, Map activeQueries) { - if (!newCrd) { - if (curCrd != null && ctx.localNodeId().equals(curCrd.nodeId()) && discoCache != null) - cleanupOrphanedServerTransactions(discoCache.serverNodes()); + /** + * Coordinator change callback. Performs all needed actions for handling new coordinator assignment. + * + * @param nodes Cluster topology snapshot. + * @param topVer Topology version. + * @param sndQrys {@code True} if it is need to send an active queries list to the new coordinator. + */ + private void onCoordinatorChanged(Collection nodes, long topVer, boolean sndQrys) { + MvccCoordinator newCrd = pickMvccCoordinator(nodes, topVer); + if (newCrd == null) return; - } - ctx.cache().context().tm().rollbackMvccTxOnCoordinatorChange(); + // Update current coordinator, collect active queries and send it to the new coordinator if needed. + GridLongList activeQryTrackers = null; - if (ctx.localNodeId().equals(curCrd.nodeId())) { - assert ctx.localNodeId().equals(curCrd.nodeId()); + synchronized (activeTrackers) { + assert curCrd == null || newCrd.topologyVersion().compareTo(curCrd.topologyVersion()) > 0; - MvccCoordinator crd = discoCache.mvccCoordinator(); + if (sndQrys) { + activeQryTrackers = new GridLongList(); - assert crd != null; + for (MvccQueryTracker tracker : activeTrackers.values()) { + long trackerId = tracker.onMvccCoordinatorChange(newCrd); - // No need to re-initialize if coordinator version hasn't changed (e.g. it was cluster activation). - if (crdVer == crd.coordinatorVersion()) - return; + if (trackerId != MVCC_TRACKER_ID_NA) + activeQryTrackers.add(trackerId); + } + } + + curCrd = newCrd; + } + + // Send local active queries to remote coordinator, if needed. + if (!newCrd.nodeId().equals(ctx.localNodeId())) { + try { + if (sndQrys) + sendMessage(newCrd.nodeId(), new MvccActiveQueriesMessage(activeQryTrackers)); + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to send active queries to mvcc coordinator: " + e); + } + } + // If a current node was elected as a new mvcc coordinator, we need to pre-initialize it. + else { + assert crdVer == 0 : crdVer; - crdVer = crd.coordinatorVersion(); + crdVer = newCrd.coordinatorVersion(); if (log.isInfoEnabled()) log.info("Initialize local node as mvcc coordinator [node=" + ctx.localNodeId() + ", crdVer=" + crdVer + ']'); - prevCrdQueries.init(activeQueries, F.view(discoCache.allNodes(), this::supportsMvcc), ctx.discovery()); + prevCrdQueries.init(activeQryTrackers, F.view(nodes, this::supportsMvcc), ctx.discovery()); - initFut.onDone(); + // Do not complete init future here, because we should wait until all old transactions become terminated. } + + crdChanged = true; } /** - * Cleans up active transacitons lost near node which is server. Executed on coordinator. + * Cleans up active transactions lost near node which is server. Executed on coordinator. + * * @param liveSrvs Live server nodes at the moment of cleanup. */ private void cleanupOrphanedServerTransactions(Collection liveSrvs) { @@ -450,24 +556,7 @@ private void cleanupOrphanedServerTransactions(Collection liveSrvs) /** {@inheritDoc} */ @Override @Nullable public MvccCoordinator currentCoordinator() { - return currentCoordinator(AffinityTopologyVersion.NONE); - } - - /** {@inheritDoc} */ - @Override @Nullable public MvccCoordinator currentCoordinator(AffinityTopologyVersion topVer) { - MvccCoordinator crd = curCrd; - - // Assert coordinator did not already change. - assert crd == null - || topVer == AffinityTopologyVersion.NONE - || crd.topologyVersion().compareTo(topVer) <= 0 : "Invalid coordinator [crd=" + crd + ", topVer=" + topVer + ']'; - - return crd; - } - - /** {@inheritDoc} */ - @Override @Nullable public MvccCoordinator assignedCoordinator() { - return assignedCrd; + return curCrd; } /** {@inheritDoc} */ @@ -477,11 +566,6 @@ private void cleanupOrphanedServerTransactions(Collection liveSrvs) return curCrd != null ? curCrd.nodeId() : null; } - /** {@inheritDoc} */ - @Override public void updateCoordinator(MvccCoordinator curCrd) { - this.curCrd = curCrd; - } - /** {@inheritDoc} */ @Override public byte state(MvccVersion ver) throws IgniteCheckedException { return state(ver.coordinatorVersion(), ver.counter()); @@ -837,51 +921,47 @@ private DataStorageConfiguration dataStorageConfiguration() { return ctx.config().getDataStorageConfiguration(); } - /** */ - private void assignMvccCoordinator(int evtType, Collection nodes, long topVer) { + /** + * Picks mvcc coordinator from the given list of nodes. + * + * @param nodes List of nodes. + * @param topVer Topology version. + * @return Chosen mvcc coordinator. + */ + private MvccCoordinator pickMvccCoordinator(Collection nodes, long topVer) { checkMvccSupported(nodes); - MvccCoordinator crd; - - if (evtType == EVT_NODE_SEGMENTED || evtType == EVT_CLIENT_NODE_DISCONNECTED) - crd = null; - else { - crd = assignedCrd; - - if (crd == null || - ((evtType == EVT_NODE_FAILED || evtType == EVT_NODE_LEFT) && !F.nodeIds(nodes).contains(crd.nodeId()))) { - ClusterNode crdNode = null; + ClusterNode crdNode = null; - if (crdC != null) { - crdNode = crdC.apply(nodes); + if (crdC != null) { + crdNode = crdC.apply(nodes); - if (log.isInfoEnabled()) - log.info("Assigned coordinator using test closure: " + crd); - } - else { - // Expect nodes are sorted by order. - for (ClusterNode node : nodes) { - if (!node.isClient() && supportsMvcc(node)) { - crdNode = node; + if (crdNode != null && log.isInfoEnabled()) + log.info("Assigned coordinator using test closure: " + crdNode.id()); + } + else { + // Expect nodes are sorted by order. + for (ClusterNode node : nodes) { + if (!node.isClient() && supportsMvcc(node)) { + crdNode = node; - break; - } - } + break; } - - crd = crdNode != null ? new MvccCoordinator(crdNode.id(), coordinatorVersion(crdNode), - new AffinityTopologyVersion(topVer, 0)) : null; - - if (log.isInfoEnabled() && crd != null) - log.info("Assigned mvcc coordinator [crd=" + crd + ", crdNode=" + crdNode + ']'); - else if (crd == null) - U.warn(log, "New mvcc coordinator was not assigned [topVer=" + topVer + ']'); } } - assignedCrd = crd; + MvccCoordinator crd = crdNode != null ? new MvccCoordinator(crdNode.id(), coordinatorVersion(crdNode), + new AffinityTopologyVersion(topVer, 0)) : null; + + if (log.isInfoEnabled() && crd != null) + log.info("Assigned mvcc coordinator [crd=" + crd + ", crdNode=" + crdNode + ']'); + else if (crd == null) + U.warn(log, "New mvcc coordinator was not assigned [topVer=" + topVer + ']'); + + return crd; } + /** * @param crdNode Assigned coordinator node. * @return Coordinator version. @@ -918,11 +998,9 @@ private boolean supportsMvcc(ClusterNode node) { } /** */ - private void checkMvccCacheStarted(@Nullable DiscoveryCustomMessage customMsg) { - assert customMsg != null; - - if (!mvccEnabled && customMsg instanceof DynamicCacheChangeBatch) { - for (DynamicCacheChangeRequest req : ((DynamicCacheChangeBatch)customMsg).requests()) { + private void checkMvccCacheStarted(DynamicCacheChangeBatch cacheMsg) { + if (!mvccEnabled) { + for (DynamicCacheChangeRequest req : cacheMsg.requests()) { CacheConfiguration ccfg = req.startCacheConfiguration(); if (ccfg == null) @@ -937,24 +1015,6 @@ private void checkMvccCacheStarted(@Nullable DiscoveryCustomMessage customMsg) { } } - /** - * @return Active queries list. - */ - private GridLongList collectActiveQueryTrackers() { - assert curCrd != null; - - GridLongList activeQryTrackers = new GridLongList(); - - for (MvccQueryTracker tracker : activeTrackers.values()) { - long trackerId = tracker.onMvccCoordinatorChange(curCrd); - - if (trackerId != MVCC_TRACKER_ID_NA) - activeQryTrackers.add(trackerId); - } - - return activeQryTrackers; - } - /** */ private MvccSnapshotResponse assignTxSnapshot(long futId, UUID nearId, boolean client) { assert initFut.isDone(); @@ -1342,7 +1402,6 @@ private void sendFutureResponse(UUID nodeId, MvccWaitTxsRequest msg) { fut.onDone(); // No need to ack, finish without error. } else - fut.onDone(e); } } @@ -1740,63 +1799,6 @@ void onNodeLeft(UUID nodeId) { } } - /** - * - */ - private class CacheCoordinatorNodeFailListener implements GridLocalEventListener { - /** {@inheritDoc} */ - @Override public void onEvent(Event evt) { - assert evt instanceof DiscoveryEvent : evt; - - DiscoveryEvent discoEvt = (DiscoveryEvent)evt; - - UUID nodeId = discoEvt.eventNode().id(); - - Map map = snapLsnrs.remove(nodeId); - - if (map != null) { - ClusterTopologyCheckedException ex = new ClusterTopologyCheckedException("Failed to request mvcc " + - "version, coordinator failed: " + nodeId); - - MvccSnapshotResponseListener lsnr; - - for (Long id : map.keySet()) { - if ((lsnr = map.remove(id)) != null) - lsnr.onError(ex); - } - } - - for (WaitAckFuture fut : ackFuts.values()) - fut.onNodeLeft(nodeId); - - activeQueries.onNodeFailed(nodeId); - - prevCrdQueries.onNodeFailed(nodeId); - - recoveryBallotBoxes.forEach((nearNodeId, ballotBox) -> { - // Put synthetic vote from another failed node - ballotBox.vote(nodeId); - - tryFinishRecoveryVoting(nearNodeId, ballotBox); - }); - - if (discoEvt.eventNode().isClient()) { - RecoveryBallotBox ballotBox = recoveryBallotBoxes - .computeIfAbsent(nodeId, uuid -> new RecoveryBallotBox()); - - ballotBox - .voters(discoEvt.topologyNodes().stream().map(ClusterNode::id).collect(Collectors.toList())); - - tryFinishRecoveryVoting(nodeId, ballotBox); - } - } - - /** {@inheritDoc} */ - @Override public String toString() { - return "CacheCoordinatorDiscoveryListener[]"; - } - } - /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java index 7296d66817b9c..d86f5ecd64f80 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java @@ -227,7 +227,7 @@ private MvccSnapshotResponseListener decorate(MvccSnapshotResponseListener lsnr) * @return {@code True} if topology is valid. */ private boolean checkTopology(AffinityTopologyVersion topVer, MvccSnapshotResponseListener lsnr) { - MvccCoordinator crd = cctx.affinity().mvccCoordinator(topVer); + MvccCoordinator crd = cctx.shared().coordinators().currentCoordinator(); if (crd == null) { lsnr.onError(noCoordinatorError(topVer)); @@ -241,16 +241,6 @@ private boolean checkTopology(AffinityTopologyVersion topVer, MvccSnapshotRespon crdVer = crd.coordinatorVersion(); } - MvccCoordinator curCrd = cctx.topology().mvccCoordinator(); - - if (!crd.equals(curCrd)) { - assert cctx.topology().topologyVersionFuture().initialVersion().compareTo(topVer) > 0; - - tryRemap(lsnr); - - return false; - } - return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java index 5c2cb0f303ca5..10b3b48bd0bd3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java @@ -267,7 +267,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter { // Wait some time in case there are some unprocessed messages from failed node. cctx.time().addTimeoutObject( - new NodeFailureTimeoutObject(evt.eventNode(), discoCache.mvccCoordinator())); + new NodeFailureTimeoutObject(evt.eventNode(), cctx.coordinators().currentCoordinator())); if (txFinishSync != null) txFinishSync.onNodeLeft(nodeId); @@ -323,7 +323,7 @@ public void rollbackOnTopologyChange(AffinityTopologyVersion topVer) { */ public void rollbackMvccTxOnCoordinatorChange() { for (IgniteInternalTx tx : activeTransactions()) { - if (tx.mvccSnapshot() != null) + if (tx.mvccSnapshot() != null && tx instanceof GridNearTxLocal) ((GridNearTxLocal)tx).rollbackNearTxLocalAsync(false, false); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java index c1718b5bc4d6a..62ea12c45d5ee 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.mvcc; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,19 +28,27 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import javax.cache.Cache; import javax.cache.CacheException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterTopologyException; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager; import org.apache.ignite.internal.processors.cache.distributed.TestCacheNodeExcludingFilter; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest; import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccSnapshotResponse; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgniteBiPredicate; @@ -675,4 +684,179 @@ protected void checkCoordinatorChangeActiveQueryClientFails_Simple(@Nullable Ign for (Ignite node : G.allGrids()) checkActiveQueriesCleanup(node); } + + /** + * @throws Exception If failed. + */ + public void testMultipleCoordinatorsLeft2Persistence() throws Exception { + persistence = true; + + checkCoordinatorsLeft(2, false); + } + + /** + * @throws Exception If failed. + */ + public void testMultipleCoordinatorsLeft3Persistence() throws Exception { + persistence = true; + + checkCoordinatorsLeft(3, true); + } + + /** + * @throws Exception If failed. + */ + public void testMultipleCoordinatorsLeft4() throws Exception { + checkCoordinatorsLeft(4, true); + } + + /** + * @param num Number of coordinators to stop. + * @throws Exception If failed. + */ + @SuppressWarnings("unchecked") + private void checkCoordinatorsLeft(int num, boolean stopCrdFirst) throws Exception { + disableScheduledVacuum = true; + + final int DATA_NODES = 3; + + final int NODES = num + DATA_NODES; + + nodeAttr = CRD_ATTR; + + // Do not use startMultithreaded here. + startGrids(num); + + nodeAttr = null; + + startGridsMultiThreaded(num, DATA_NODES); + + List victims = new ArrayList<>(num); + List survivors = new ArrayList<>(DATA_NODES); + + for (int i = 0; i < NODES; i++) { + if (i < num) + victims.add(grid(i)); + else + survivors.add(grid(i)); + } + + if (log.isInfoEnabled()) { + log.info("Nodes to be stopped [" + + victims.stream(). + map(n -> n.cluster().localNode().id().toString()) + .collect(Collectors.joining(", ")) + ']'); + + log.info("Nodes not to be stopped [" + + survivors.stream(). + map(n -> n.cluster().localNode().id().toString()) + .collect(Collectors.joining(", ")) + ']'); + } + + Ignite nearNode = survivors.get(0); + + if (persistence) + nearNode.cluster().active(true); + + CacheConfiguration ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, DATA_NODES - 1, DFLT_PARTITION_COUNT) + .setNodeFilter(new CoordinatorNodeFilter()); + + IgniteCache cache = nearNode.createCache(ccfg); + + try (Transaction tx = nearNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + for (int key = 0; key < 10; key++) + cache.put(key, 1); + + tx.commit(); + } + + List stopThreads = victims.stream() + .map(v -> new Thread(() -> stopGrid(v.name()))) + .collect(Collectors.toList()); + + ScanQuery scan = new ScanQuery<>(); + + QueryCursor> cur = survivors.get(0).cache(DEFAULT_CACHE_NAME).query(scan); + + Iterator> it = cur.iterator(); + + assertTrue(it.hasNext()); + assertEquals(1, it.next().getValue()); + + if (log.isInfoEnabled()) + log.info("Start stopping nodes."); + + // Stop nodes and join threads. + if (stopCrdFirst) { + for (Thread t : stopThreads) + t.start(); + } + else { + // We should stop the oldest node last. + GridCachePartitionExchangeManager exch = ((IgniteEx)survivors.get(1)).context().cache().context().exchange(); + + GridDhtTopologyFuture lastFinished = exch.lastFinishedFuture(); + + for (int i = 1; i < stopThreads.size(); i++) + stopThreads.get(i).start(); + + while (lastFinished == exch.lastTopologyFuture()) + doSleep(1); + + stopThreads.get(0).start(); + } + + for (Thread t : stopThreads) + t.join(); + + if (log.isInfoEnabled()) + log.info("All nodes stopped."); + + assertTrue(it.hasNext()); + assertEquals(1, it.next().getValue()); + + for (Ignite node : survivors) { + for (int key = 0; key < 10; key++) + assertEquals(1, node.cache(DEFAULT_CACHE_NAME).get(key)); + } + + try (Transaction tx = nearNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + for (int key = 0; key < 10; key++) + cache.put(key, 2); + + tx.commit(); + } + catch (Exception e) { + stopAllGrids(true); + + fail(X.getFullStackTrace(e)); + } + + for (Ignite node : survivors) { + for (int key = 0; key < 10; key++) + assertEquals(2, node.cache(DEFAULT_CACHE_NAME).get(key)); + } + + try (Transaction tx = nearNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + for (int key = 0; key < 10; key++) + cache.put(key, 3); + + tx.commit(); + } + catch (Exception e) { + stopAllGrids(true); + + fail(X.getFullStackTrace(e)); + } + + for (Ignite node : survivors) { + for (int key = 0; key < 10; key++) + assertEquals(3, node.cache(DEFAULT_CACHE_NAME).get(key)); + } + + while (it.hasNext()) + assertEquals(1, (int)it.next().getValue()); + + cur.close(); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedCoordinatorFailoverTest.java index 3ea1c5bb35317..342538210aef8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedCoordinatorFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccPartitionedCoordinatorFailoverTest.java @@ -17,8 +17,12 @@ package org.apache.ignite.internal.processors.cache.mvcc; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.GET; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.PUT; @@ -141,4 +145,35 @@ public void testGetReadInsideTxInProgressCoordinatorFails_ReadDelay() throws Exc public void testReadInProgressCoordinatorFailsSimple_FromServerPutGet() throws Exception { readInProgressCoordinatorFailsSimple(false, null, GET, PUT); } + + /** + * @throws Exception If failed. + */ + public void testActivateDeactivateCLuster() throws Exception { + disableScheduledVacuum = true; + persistence = true; + + final int DATA_NODES = 3; + + // Do not use startMultithreaded here. + startGrids(DATA_NODES); + + Ignite near = grid(DATA_NODES - 1); + + CacheConfiguration ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, DATA_NODES - 1, DFLT_PARTITION_COUNT); + + near.cluster().active(true); + + IgniteCache cache = near.createCache(ccfg); + + cache.put(1, 1); + + near.cluster().active(false); + + stopGrid(0); + + near.cluster().active(true); + + assertEquals(1, cache.get(1)); + } } From 5afb9d955c9b218a8c1971761e8f3b09ac48bc82 Mon Sep 17 00:00:00 2001 From: vd-pyatkov Date: Fri, 23 Nov 2018 14:05:46 +0300 Subject: [PATCH 164/403] IGNITE-10323 Don't run deactivation process on inactive cluster - Fixes #5476. Signed-off-by: Pavel Kovalenko (cherry picked from commit 39b59d21e96bac51dca9a12702f1993ad96fead9) --- .../cluster/GridClusterStateProcessor.java | 3 +- ...ActivateDeactivateTestWithPersistence.java | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index e0a4f0a4528fd..353d4e36cbe47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -850,7 +850,8 @@ private IgniteInternalFuture changeGlobalState0(final boolean activate, DiscoveryDataClusterState curState = globalState; - if (!curState.transition() && curState.active() == activate && BaselineTopology.equals(curState.baselineTopology(), blt)) + if (!curState.transition() && curState.active() == activate + && (!activate || BaselineTopology.equals(curState.baselineTopology(), blt))) return new GridFinishedFuture<>(); GridChangeGlobalStateFuture startedFut = null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java index 0972ea2a8f6ab..4dc4f2743371b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java @@ -98,6 +98,43 @@ public void testActivateCachesRestore_5_Servers_WithNewCaches() throws Exception activateCachesRestore(5, true); } + /** + * Test deactivation on cluster that is not yet activated. + * + * @throws Exception If failed. + */ + public void testDeactivateInactiveCluster() throws Exception { + ccfgs = new CacheConfiguration[] { + new CacheConfiguration<>("test_cache_1") + .setGroupName("test_cache"), + new CacheConfiguration<>("test_cache_2") + .setGroupName("test_cache") + }; + + Ignite ignite = startGrids(3); + + ignite.cluster().active(true); + + ignite.cache("test_cache_1") + .put("key1", "val1"); + ignite.cache("test_cache_2") + .put("key1", "val1"); + + ignite.cluster().active(false); + + assertFalse(ignite.cluster().active()); + + stopAllGrids(); + + ignite = startGrids(2); + + assertFalse(ignite.cluster().active()); + + ignite.cluster().active(false); + + assertFalse(ignite.cluster().active()); + } + /** */ private Map startGridsAndLoadData(int srvs) throws Exception { Ignite srv = startGrids(srvs); From aacb14a2bab8e27746812b89d8d65660d00ac000 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Fri, 23 Nov 2018 19:11:31 +0300 Subject: [PATCH 165/403] IGNITE-10381 Fixed U.doInParallel not to terminate early - Fixes #5483. Signed-off-by: Alexey Goncharuk --- .../GridCachePartitionExchangeManager.java | 4 +- .../GridDhtPartitionsExchangeFuture.java | 3 +- .../ignite/internal/util/IgniteUtils.java | 189 +++--------------- ...amicCacheStartFailWithPersistenceTest.java | 1 + .../internal/util/IgniteUtilsSelfTest.java | 43 ---- 5 files changed, 34 insertions(+), 206 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 525cb88317395..81f8b8c8af204 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -2901,8 +2901,8 @@ else if (task instanceof ForceRebalanceExchangeTask) { "topVer=" + exchFut.initialVersion() + ", node=" + cctx.localNodeId() + "]. " + (curTimeout <= 0 && !txRolledBack ? "Consider changing " + - "TransactionConfiguration.txTimeoutOnPartitionMapSynchronization" + - " to non default value to avoid this message. " : "") + + "TransactionConfiguration.txTimeoutOnPartitionMapSynchronization" + + " to non default value to avoid this message. " : "") + "Dumping pending objects that might be the cause: "); try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index da1a25beb7afe..31ce07e9d5ef7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -1191,7 +1191,8 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep // This exception will be handled by init() method. throw e; - U.error(log, "Failed to initialize cache(s) (will try to rollback). " + exchId, e); + U.error(log, "Failed to initialize cache(s) (will try to rollback) [exchId=" + exchId + + ", caches=" + exchActions.cacheGroupsToStart() + ']', e); exchangeLocE = new IgniteCheckedException( "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 4080924883715..fcb99782bbe46 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -202,7 +202,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -221,7 +220,6 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.StringTokenizer; -import java.util.TimeZone; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.BrokenBarrierException; @@ -7198,137 +7196,6 @@ private static boolean checkNextToken(StringTokenizer t, String str, String date } } - /** - * - * @param str ISO date. - * @return Calendar instance. - * @throws IgniteCheckedException Thrown in case of any errors. - */ - public static Calendar parseIsoDate(String str) throws IgniteCheckedException { - StringTokenizer t = new StringTokenizer(str, "+-:.TZ", true); - - Calendar cal = Calendar.getInstance(); - cal.clear(); - - try { - if (t.hasMoreTokens()) - cal.set(Calendar.YEAR, Integer.parseInt(t.nextToken())); - else - return cal; - - if (checkNextToken(t, "-", str) && t.hasMoreTokens()) - cal.set(Calendar.MONTH, Integer.parseInt(t.nextToken()) - 1); - else - return cal; - - if (checkNextToken(t, "-", str) && t.hasMoreTokens()) - cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(t.nextToken())); - else - return cal; - - if (checkNextToken(t, "T", str) && t.hasMoreTokens()) - cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(t.nextToken())); - else { - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - - return cal; - } - - if (checkNextToken(t, ":", str) && t.hasMoreTokens()) - cal.set(Calendar.MINUTE, Integer.parseInt(t.nextToken())); - else { - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - - return cal; - } - - if (!t.hasMoreTokens()) - return cal; - - String tok = t.nextToken(); - - if (":".equals(tok)) { // Seconds. - if (t.hasMoreTokens()) { - cal.set(Calendar.SECOND, Integer.parseInt(t.nextToken())); - - if (!t.hasMoreTokens()) - return cal; - - tok = t.nextToken(); - - if (".".equals(tok)) { - String nt = t.nextToken(); - - while (nt.length() < 3) - nt += "0"; - - nt = nt.substring(0, 3); // Cut trailing chars. - - cal.set(Calendar.MILLISECOND, Integer.parseInt(nt)); - - if (!t.hasMoreTokens()) - return cal; - - tok = t.nextToken(); - } - else - cal.set(Calendar.MILLISECOND, 0); - } - else - throw new IgniteCheckedException("Invalid date format: " + str); - } - else { - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - } - - if (!"Z".equals(tok)) { - if (!"+".equals(tok) && !"-".equals(tok)) - throw new IgniteCheckedException("Invalid date format: " + str); - - boolean plus = "+".equals(tok); - - if (!t.hasMoreTokens()) - throw new IgniteCheckedException("Invalid date format: " + str); - - tok = t.nextToken(); - - int tzHour; - int tzMin; - - if (tok.length() == 4) { - tzHour = Integer.parseInt(tok.substring(0, 2)); - tzMin = Integer.parseInt(tok.substring(2, 4)); - } - else { - tzHour = Integer.parseInt(tok); - - if (checkNextToken(t, ":", str) && t.hasMoreTokens()) - tzMin = Integer.parseInt(t.nextToken()); - else - throw new IgniteCheckedException("Invalid date format: " + str); - } - - if (plus) - cal.set(Calendar.ZONE_OFFSET, (tzHour * 60 + tzMin) * 60 * 1000); - else - cal.set(Calendar.ZONE_OFFSET, -(tzHour * 60 + tzMin) * 60 * 1000); - } - else - cal.setTimeZone(TimeZone.getTimeZone("GMT")); - } - catch (NumberFormatException ex) { - throw new IgniteCheckedException("Invalid date format: " + str, ex); - } - - return cal; - } - /** * Adds values to collection and returns the same collection to allow chaining. * @@ -7434,20 +7301,6 @@ public static IgniteCheckedException cast(Throwable t) { : new IgniteCheckedException(t); } - /** - * Parses passed string with specified date. - * - * @param src String to parse. - * @param ptrn Pattern. - * @return Parsed date. - * @throws java.text.ParseException If exception occurs while parsing. - */ - public static Date parse(String src, String ptrn) throws java.text.ParseException { - java.text.DateFormat format = new java.text.SimpleDateFormat(ptrn); - - return format.parse(src); - } - /** * Checks if class loader is an internal P2P class loader. * @@ -10855,7 +10708,7 @@ public static Collection doInParallel( batch.result(res); } - catch (IgniteCheckedException e) { + catch (Throwable e) { batch.result(e); } } @@ -10868,10 +10721,7 @@ public static Collection doInParallel( Throwable err = batch.error; if (err != null) { - if (error == null) - error = err; - else - error.addSuppressed(err); + error = addSuppressed(error, err); continue; } @@ -10889,16 +10739,10 @@ public static Collection doInParallel( throw new IgniteInterruptedCheckedException(e); } catch (ExecutionException e) { - if(error == null) - error = e.getCause(); - else - error.addSuppressed(e.getCause()); + error = addSuppressed(error, e.getCause()); } catch (CancellationException e) { - if(error == null) - error = e; - else - error.addSuppressed(e); + error = addSuppressed(error, e); } } @@ -10918,6 +10762,31 @@ public static Collection doInParallel( return results; } + /** + * Utility method to add the given throwable error to the given throwable root error. If the given + * suppressed throwable is an {@code Error}, but the root error is not, will change the root to the {@code Error}. + * + * @param root Root error to add suppressed error to. + * @param err Error to add. + * @return New root error. + */ + private static Throwable addSuppressed(Throwable root, Throwable err) { + assert err != null; + + if (root == null) + return err; + + if (err instanceof Error && !(root instanceof Error)) { + err.addSuppressed(root); + + root = err; + } + else + root.addSuppressed(err); + + return root; + } + /** * The batch of tasks with a batch index in global array. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java index 24c9342be20ce..8643b66a19ba1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java @@ -34,6 +34,7 @@ public class IgniteDynamicCacheStartFailWithPersistenceTest extends IgniteAbstra return 5 * 60 * 1000; } + /** {@inheritDoc} */ @Override protected boolean persistenceEnabled() { return true; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java index 73fb9cc9839e7..85b8b4a956529 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java @@ -38,7 +38,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -401,48 +400,6 @@ private static ComputeJob job(final Runnable r) { }; } - /** - * - * @throws Exception If failed. - */ - public void testParseIsoDate() throws Exception { - Calendar cal = U.parseIsoDate("2009-12-08T13:30:44.000Z"); - - assert cal.get(Calendar.YEAR) == 2009; - assert cal.get(Calendar.MONTH) == 11; - assert cal.get(Calendar.DAY_OF_MONTH) == 8; - assert cal.get(Calendar.HOUR_OF_DAY) == 13; - assert cal.get(Calendar.MINUTE) == 30; - assert cal.get(Calendar.SECOND) == 44; - assert cal.get(Calendar.MILLISECOND) == 0; - assert cal.get(Calendar.ZONE_OFFSET) == 0 : - "Unexpected value: " + cal.get(Calendar.ZONE_OFFSET); - - cal = U.parseIsoDate("2009-12-08T13:30:44.000+03:00"); - - assert cal.get(Calendar.YEAR) == 2009; - assert cal.get(Calendar.MONTH) == 11; - assert cal.get(Calendar.DAY_OF_MONTH) == 8; - assert cal.get(Calendar.HOUR_OF_DAY) == 13; - assert cal.get(Calendar.MINUTE) == 30; - assert cal.get(Calendar.SECOND) == 44; - assert cal.get(Calendar.MILLISECOND) == 0; - assert cal.get(Calendar.ZONE_OFFSET) == 3 * 60 * 60 * 1000 : - "Unexpected value: " + cal.get(Calendar.ZONE_OFFSET); - - cal = U.parseIsoDate("2009-12-08T13:30:44.000+0300"); - - assert cal.get(Calendar.YEAR) == 2009; - assert cal.get(Calendar.MONTH) == 11; - assert cal.get(Calendar.DAY_OF_MONTH) == 8; - assert cal.get(Calendar.HOUR_OF_DAY) == 13; - assert cal.get(Calendar.MINUTE) == 30; - assert cal.get(Calendar.SECOND) == 44; - assert cal.get(Calendar.MILLISECOND) == 0; - assert cal.get(Calendar.ZONE_OFFSET) == 3 * 60 * 60 * 1000 : - "Unexpected value: " + cal.get(Calendar.ZONE_OFFSET); - } - /** * @throws Exception If test failed. */ From 6f67efe842a1d9489b910820828afb2d0c5c105b Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Fri, 23 Nov 2018 20:23:16 +0300 Subject: [PATCH 166/403] IGNITE-10392 Use lasdAffChangedTopVer if DiscoCache for the requested topVer isn't ready - Fixes #5488. Signed-off-by: Alexey Goncharuk (cherry picked from commit 5c01c4199df3ba3a479d4566561335d57adffd0f) --- .../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index 52c484415dee0..e04b8d704daf0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -2058,6 +2058,14 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer) snap.discoCache : discoCacheHist.get(topVer); if (cache == null) { + AffinityTopologyVersion lastAffChangedTopVer = + ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer); + + DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer); + + if (lastAffChangedDiscoCache != null) + return lastAffChangedDiscoCache; + CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId); throw new IgniteException("Failed to resolve nodes topology [" + From 3a273680c63582e65b365ad3eaeb374d378d0af6 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 23 Nov 2018 21:22:12 +0300 Subject: [PATCH 167/403] IGNITE-9996: Performance drop final fix. --- .../persistence/pagemem/PageMemoryImpl.java | 6 +++--- .../wal/serializer/RecordDataV1Serializer.java | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index f89a2871ceaee..988b9364c0f15 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -228,7 +228,7 @@ public class PageMemoryImpl implements PageMemoryEx { private final GridEncryptionManager encMgr; /** */ - private final EncryptionSpi encSpi; + private final boolean encryptionDisabled; /** */ private final IgniteLogger log; @@ -324,7 +324,7 @@ public PageMemoryImpl( storeMgr = ctx.pageStore(); walMgr = ctx.wal(); encMgr = ctx.kernalContext().encryption(); - encSpi = ctx.gridConfig().getEncryptionSpi(); + encryptionDisabled = ctx.gridConfig().getEncryptionSpi() instanceof NoopEncryptionSpi; assert storeMgr != null; assert walMgr != null; @@ -974,7 +974,7 @@ private void tryToRestorePage(FullPageId fullId, ByteBuffer buf) throws IgniteCh /** {@inheritDoc} */ @Override public int realPageSize(int grpId) { - if ((encSpi instanceof NoopEncryptionSpi) || encMgr.groupKey(grpId) == null) + if (encryptionDisabled || encMgr.groupKey(grpId) == null) return pageSize(); return encPageSize; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 99032b04cc67d..46f36119fb83b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -147,6 +147,9 @@ public class RecordDataV1Serializer implements RecordDataSerializer { /** Encryption manager. */ private final GridEncryptionManager encMgr; + /** */ + private final boolean encryptionDisabled; + /** */ private static final byte ENCRYPTED = 1; @@ -164,6 +167,8 @@ public RecordDataV1Serializer(GridCacheSharedContext cctx) { this.encSpi = cctx.gridConfig().getEncryptionSpi(); this.encMgr = cctx.kernalContext().encryption(); + encryptionDisabled = encSpi instanceof NoopEncryptionSpi; + //This happen on offline WAL iteration(we don't have encryption keys available). if (encSpi != null) this.realPageSize = CU.encryptedPageSize(pageSize, encSpi); @@ -229,7 +234,7 @@ public RecordDataV1Serializer(GridCacheSharedContext cctx) { * @return {@code True} if this record should be encrypted. */ private boolean needEncryption(WALRecord rec) { - if (encSpi instanceof NoopEncryptionSpi) + if (encryptionDisabled) return false; if (!(rec instanceof WalRecordCacheGroupAware)) @@ -243,7 +248,7 @@ private boolean needEncryption(WALRecord rec) { * @return {@code True} if this record should be encrypted. */ private boolean needEncryption(int grpId) { - if (encSpi instanceof NoopEncryptionSpi) + if (encryptionDisabled) return false; return encMgr != null && encMgr.groupKey(grpId) != null; @@ -1924,6 +1929,9 @@ DataEntry readPlainDataEntry(ByteBufferBackedDataInput in) throws IOException, I * @return Real record type. */ RecordType recordType(WALRecord rec) { + if (encryptionDisabled) + return rec.type(); + if (needEncryption(rec)) return ENCRYPTED_RECORD; @@ -1938,6 +1946,9 @@ RecordType recordType(WALRecord rec) { * @return {@code True} if this data record should be encrypted. */ boolean isDataRecordEncrypted(DataRecord rec) { + if (encryptionDisabled) + return false; + for (DataEntry e : rec.writeEntries()) { if (cctx.cacheContext(e.cacheId()) != null && needEncryption(cctx.cacheContext(e.cacheId()).groupId())) return true; @@ -2015,7 +2026,7 @@ protected int dataSize(DataRecord dataRec) throws IgniteCheckedException { for (DataEntry entry : dataRec.writeEntries()) { int clSz = entrySize(entry); - if (needEncryption(cctx.cacheContext(entry.cacheId()).groupId())) + if (!encryptionDisabled && needEncryption(cctx.cacheContext(entry.cacheId()).groupId())) sz += encSpi.encryptedSize(clSz) + 1 /* encrypted flag */ + 4 /* groupId */ + 4 /* data size */; else { sz += clSz; From adf00ca849a908f8e9ed5a40b102740e3f98fa35 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 23 Nov 2018 22:51:08 +0300 Subject: [PATCH 168/403] Revert "IGNITE-8493 GridToStringBuilder arrayToString refactoring. - Fixes #4501." This reverts commit dd8c933fd44e4ad9b315daccce8e2327606867b0. --- .../util/tostring/GridToStringBuilder.java | 129 +++++++++++------- .../tostring/GridToStringBuilderSelfTest.java | 97 +++++-------- ...teHadoopFileSystemClientBasedOpenTest.java | 2 +- 3 files changed, 112 insertions(+), 116 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java index 329682f462525..67b9af12d2bc3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java @@ -21,7 +21,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; -import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.ArrayList; @@ -854,7 +853,7 @@ else if (val instanceof Map) */ private static void addArray(SBLimitedLength buf, Class arrType, Object obj) { if (arrType.getComponentType().isPrimitive()) { - buf.a(arrayToString(obj)); + buf.a(arrayToString(arrType, obj)); return; } @@ -1076,40 +1075,95 @@ public static String toString(String str, String name, @Nullable Object val) { } /** - * Returns limited string representation of array. - * - * @param arr Array object. Each value is automatically wrapped if it has a primitive type. + * @param arrType Type of the array. + * @param arr Array object. * @return String representation of an array. */ - public static String arrayToString(Object arr) { + @SuppressWarnings({"ConstantConditions", "unchecked"}) + public static String arrayToString(Class arrType, Object arr) { if (arr == null) return "null"; String res; + int more = 0; - int arrLen; - - if (arr instanceof Object[]) { + if (arrType.equals(byte[].class)) { + byte[] byteArr = (byte[])arr; + if (byteArr.length > COLLECTION_LIMIT) { + more = byteArr.length - COLLECTION_LIMIT; + byteArr = Arrays.copyOf(byteArr, COLLECTION_LIMIT); + } + res = Arrays.toString(byteArr); + } + else if (arrType.equals(boolean[].class)) { + boolean[] boolArr = (boolean[])arr; + if (boolArr.length > COLLECTION_LIMIT) { + more = boolArr.length - COLLECTION_LIMIT; + boolArr = Arrays.copyOf(boolArr, COLLECTION_LIMIT); + } + res = Arrays.toString(boolArr); + } + else if (arrType.equals(short[].class)) { + short[] shortArr = (short[])arr; + if (shortArr.length > COLLECTION_LIMIT) { + more = shortArr.length - COLLECTION_LIMIT; + shortArr = Arrays.copyOf(shortArr, COLLECTION_LIMIT); + } + res = Arrays.toString(shortArr); + } + else if (arrType.equals(int[].class)) { + int[] intArr = (int[])arr; + if (intArr.length > COLLECTION_LIMIT) { + more = intArr.length - COLLECTION_LIMIT; + intArr = Arrays.copyOf(intArr, COLLECTION_LIMIT); + } + res = Arrays.toString(intArr); + } + else if (arrType.equals(long[].class)) { + long[] longArr = (long[])arr; + if (longArr.length > COLLECTION_LIMIT) { + more = longArr.length - COLLECTION_LIMIT; + longArr = Arrays.copyOf(longArr, COLLECTION_LIMIT); + } + res = Arrays.toString(longArr); + } + else if (arrType.equals(float[].class)) { + float[] floatArr = (float[])arr; + if (floatArr.length > COLLECTION_LIMIT) { + more = floatArr.length - COLLECTION_LIMIT; + floatArr = Arrays.copyOf(floatArr, COLLECTION_LIMIT); + } + res = Arrays.toString(floatArr); + } + else if (arrType.equals(double[].class)) { + double[] doubleArr = (double[])arr; + if (doubleArr.length > COLLECTION_LIMIT) { + more = doubleArr.length - COLLECTION_LIMIT; + doubleArr = Arrays.copyOf(doubleArr, COLLECTION_LIMIT); + } + res = Arrays.toString(doubleArr); + } + else if (arrType.equals(char[].class)) { + char[] charArr = (char[])arr; + if (charArr.length > COLLECTION_LIMIT) { + more = charArr.length - COLLECTION_LIMIT; + charArr = Arrays.copyOf(charArr, COLLECTION_LIMIT); + } + res = Arrays.toString(charArr); + } + else { Object[] objArr = (Object[])arr; - - arrLen = objArr.length; - - if (arrLen > COLLECTION_LIMIT) + if (objArr.length > COLLECTION_LIMIT) { + more = objArr.length - COLLECTION_LIMIT; objArr = Arrays.copyOf(objArr, COLLECTION_LIMIT); - + } res = Arrays.toString(objArr); - } else { - res = toStringWithLimit(arr, COLLECTION_LIMIT); - - arrLen = Array.getLength(arr); } - - if (arrLen > COLLECTION_LIMIT) { + if (more > 0) { StringBuilder resSB = new StringBuilder(res); resSB.deleteCharAt(resSB.length() - 1); - - resSB.append("... and ").append(arrLen - COLLECTION_LIMIT).append(" more]"); + resSB.append("... and ").append(more).append(" more]"); res = resSB.toString(); } @@ -1117,37 +1171,6 @@ public static String arrayToString(Object arr) { return res; } - /** - * Returns limited string representation of array. - * - * @param arr Input array. Each value is automatically wrapped if it has a primitive type. - * @param limit max array items to string limit. - * @return String representation of an array. - */ - private static String toStringWithLimit(Object arr, int limit) { - int arrIdxMax = Array.getLength(arr) - 1; - - if (arrIdxMax == -1) - return "[]"; - - int idxMax = Math.min(arrIdxMax, limit); - - StringBuilder b = new StringBuilder(); - - b.append('['); - - for (int i = 0; i <= idxMax; ++i) { - b.append(Array.get(arr, i)); - - if (i == idxMax) - return b.append(']').toString(); - - b.append(", "); - } - - return b.toString(); - } - /** * Produces uniformed output of string with context properties * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java index d249914ccb3bc..9a9e67e60f905 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java @@ -304,8 +304,8 @@ public void testArrLimitWithRecursion() throws Exception { * @param limit Array limit. */ private void checkArrayOverflow(Object[] arrOf, Object[] arr, int limit) { - String arrStr = GridToStringBuilder.arrayToString(arr); - String arrOfStr = GridToStringBuilder.arrayToString(arrOf); + String arrStr = GridToStringBuilder.arrayToString(arr.getClass(), arr); + String arrOfStr = GridToStringBuilder.arrayToString(arrOf.getClass(), arrOf); // Simulate overflow StringBuilder resultSB = new StringBuilder(arrStr); @@ -332,97 +332,75 @@ public void testToStringCollectionLimits() throws Exception { for (Object val : vals) testArr(val, limit); - int[] intArr1 = new int[0]; - - assertEquals("[]", GridToStringBuilder.arrayToString(intArr1)); - assertEquals("null", GridToStringBuilder.arrayToString(null)); - - int[] intArr2 = {1, 2, 3}; - - assertEquals("[1, 2, 3]", GridToStringBuilder.arrayToString(intArr2)); - - Object[] intArr3 = {2, 3, 4}; - - assertEquals("[2, 3, 4]", GridToStringBuilder.arrayToString(intArr3)); - byte[] byteArr = new byte[1]; - byteArr[0] = 1; - assertEquals(Arrays.toString(byteArr), GridToStringBuilder.arrayToString(byteArr)); + assertEquals(Arrays.toString(byteArr), GridToStringBuilder.arrayToString(byteArr.getClass(), byteArr)); byteArr = Arrays.copyOf(byteArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(byteArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(byteArr.getClass(), byteArr).contains("... and 1 more")); boolean[] boolArr = new boolean[1]; - boolArr[0] = true; - assertEquals(Arrays.toString(boolArr), GridToStringBuilder.arrayToString(boolArr)); + assertEquals(Arrays.toString(boolArr), GridToStringBuilder.arrayToString(boolArr.getClass(), boolArr)); boolArr = Arrays.copyOf(boolArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(boolArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(boolArr.getClass(), boolArr).contains("... and 1 more")); short[] shortArr = new short[1]; - shortArr[0] = 100; - assertEquals(Arrays.toString(shortArr), GridToStringBuilder.arrayToString(shortArr)); + assertEquals(Arrays.toString(shortArr), GridToStringBuilder.arrayToString(shortArr.getClass(), shortArr)); shortArr = Arrays.copyOf(shortArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(shortArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(shortArr.getClass(), shortArr).contains("... and 1 more")); int[] intArr = new int[1]; - intArr[0] = 10000; - assertEquals(Arrays.toString(intArr), GridToStringBuilder.arrayToString(intArr)); + assertEquals(Arrays.toString(intArr), GridToStringBuilder.arrayToString(intArr.getClass(), intArr)); intArr = Arrays.copyOf(intArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(intArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(intArr.getClass(), intArr).contains("... and 1 more")); long[] longArr = new long[1]; - longArr[0] = 10000000; - assertEquals(Arrays.toString(longArr), GridToStringBuilder.arrayToString(longArr)); + assertEquals(Arrays.toString(longArr), GridToStringBuilder.arrayToString(longArr.getClass(), longArr)); longArr = Arrays.copyOf(longArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(longArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(longArr.getClass(), longArr).contains("... and 1 more")); float[] floatArr = new float[1]; - floatArr[0] = 1.f; - assertEquals(Arrays.toString(floatArr), GridToStringBuilder.arrayToString(floatArr)); + assertEquals(Arrays.toString(floatArr), GridToStringBuilder.arrayToString(floatArr.getClass(), floatArr)); floatArr = Arrays.copyOf(floatArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(floatArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(floatArr.getClass(), floatArr).contains("... and 1 more")); double[] doubleArr = new double[1]; - doubleArr[0] = 1.; - assertEquals(Arrays.toString(doubleArr), GridToStringBuilder.arrayToString(doubleArr)); + assertEquals(Arrays.toString(doubleArr), GridToStringBuilder.arrayToString(doubleArr.getClass(), doubleArr)); doubleArr = Arrays.copyOf(doubleArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(doubleArr).contains("... and 1 more")); - - char[] cArr = new char[1]; + GridToStringBuilder.arrayToString(doubleArr.getClass(), doubleArr).contains("... and 1 more")); - cArr[0] = 'a'; - assertEquals(Arrays.toString(cArr), GridToStringBuilder.arrayToString(cArr)); - cArr = Arrays.copyOf(cArr, 101); + char[] charArr = new char[1]; + charArr[0] = 'a'; + assertEquals(Arrays.toString(charArr), GridToStringBuilder.arrayToString(charArr.getClass(), charArr)); + charArr = Arrays.copyOf(charArr, 101); assertTrue("Can't find \"... and 1 more\" in overflowed array string!", - GridToStringBuilder.arrayToString(cArr).contains("... and 1 more")); + GridToStringBuilder.arrayToString(charArr.getClass(), charArr).contains("... and 1 more")); Map strMap = new TreeMap<>(); List strList = new ArrayList<>(limit+1); - TestClass1 testCls = new TestClass1(); - - testCls.strMap = strMap; - testCls.strListIncl = strList; + TestClass1 testClass = new TestClass1(); + testClass.strMap = strMap; + testClass.strListIncl = strList; for (int i = 0; i < limit; i++) { strMap.put("k" + i, "v"); strList.add("e"); } - checkColAndMap(testCls); + checkColAndMap(testClass); } /** @@ -481,26 +459,21 @@ private void checkColAndMap(TestClass1 testCls) { public void testToStringSizeLimits() throws Exception { int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_MAX_LENGTH, 10_000); int tailLen = limit / 10 * 2; - StringBuilder sb = new StringBuilder(limit + 10); - - for (int i = 0; i < limit - 100; i++) + for (int i = 0; i < limit - 100; i++) { sb.append('a'); - + } String actual = GridToStringBuilder.toString(TestClass2.class, new TestClass2(sb.toString())); - String exp = "TestClass2 [str=" + sb + ", nullArr=null]"; + String expected = "TestClass2 [str=" + sb.toString() + ", nullArr=null]"; + assertEquals(expected, actual); - assertEquals(exp, actual); - - for (int i = 0; i < 110; i++) + for (int i = 0; i < 110; i++) { sb.append('b'); - + } actual = GridToStringBuilder.toString(TestClass2.class, new TestClass2(sb.toString())); - exp = "TestClass2 [str=" + sb + ", nullArr=null]"; - - assertEquals(exp.substring(0, limit - tailLen), actual.substring(0, limit - tailLen)); - assertEquals(exp.substring(exp.length() - tailLen), actual.substring(actual.length() - tailLen)); - + expected = "TestClass2 [str=" + sb.toString() + ", nullArr=null]"; + assertEquals(expected.substring(0, limit - tailLen), actual.substring(0, limit - tailLen)); + assertEquals(expected.substring(expected.length() - tailLen), actual.substring(actual.length() - tailLen)); assertTrue(actual.contains("... and")); assertTrue(actual.contains("skipped ...")); } @@ -647,7 +620,7 @@ private static class TestClass2{ /** * @param str String. */ - TestClass2(String str) { + public TestClass2(String str) { this.str = str; } } diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java index 932f4d81dfb22..289e63dbccf82 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java @@ -176,7 +176,7 @@ public void testFsOpenMultithreaded() throws Exception { */ private void checkFsOpenWithAllNodesTypes() throws Exception { for (int i = 0; i < nodesTypes.length; ++i) { - log.info("Begin test case for nodes: " + S.arrayToString(nodesTypes[i])); + log.info("Begin test case for nodes: " + S.arrayToString(NodeType.class, nodesTypes[i])); startNodes(nodesTypes[i]); From eb0e40c987cb79fe53150a0cd056cf347feb0c65 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 23 Nov 2018 22:51:45 +0300 Subject: [PATCH 169/403] Revert "IGNITE-9209 fix for GridDistributedTxMapping.toString() returns broken string. - Fixes #4519." This reverts commit 9bb9c043513c3cc6c6b70c6c3395e5bb76fad75e. --- .../internal/util/tostring/GridToStringBuilder.java | 10 +++------- .../util/tostring/GridToStringBuilderSelfTest.java | 13 ------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java index 67b9af12d2bc3..86daf7cc7b21e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java @@ -985,7 +985,6 @@ private static String toStringImpl( if (newStr) return s; - // Called from another GTSB.toString(), so this string is already in the buffer and shouldn't be returned. return ""; } finally { @@ -1053,6 +1052,7 @@ private static String toStringImpl0( } // Specifically catching all exceptions. catch (Exception e) { + // Remove entry from cache to avoid potential memory leak // in case new class loader got loaded under the same identity hash. classCache.remove(cls.getName() + System.identityHashCode(cls.getClassLoader())); @@ -1605,7 +1605,7 @@ public static String toString(String str, private static String toStringImpl(String str, SBLimitedLength buf, Object[] propNames, Object[] propVals, boolean[] propSens, int propCnt) { - boolean newStr = buf.length() == 0; + buf.setLength(0); if (str != null) buf.a(str).a(" "); @@ -1616,11 +1616,7 @@ private static String toStringImpl(String str, SBLimitedLength buf, Object[] pro buf.a(']'); - if (newStr) - return buf.toString(); - - // Called from another GTSB.toString(), so this string is already in the buffer and shouldn't be returned. - return ""; + return buf.toString(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java index 9a9e67e60f905..eff3349f37e44 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java @@ -31,8 +31,6 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; -import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -478,17 +476,6 @@ public void testToStringSizeLimits() throws Exception { assertTrue(actual.contains("skipped ...")); } - /** - * - */ - public void testObjectPlusStringToString() { - IgniteTxKey k = new IgniteTxKey(new KeyCacheObjectImpl(1, null, 1), 123); - - info(k.toString()); - - assertTrue("Wrong string: " + k, k.toString().startsWith("IgniteTxKey [")); - } - /** * Test class. */ From adf10876e322ee200219abe2e19e21b2043d5072 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 23 Nov 2018 22:52:12 +0300 Subject: [PATCH 170/403] Revert "IGNITE-602 Fixed possible StackOverflowError in GridToStringBuilder when circular references are present - Fixes #1558." This reverts commit d67c5bf4c22338695a116e0fbf0a58a4d581af5d. --- .../util/tostring/GridToStringBuilder.java | 706 +++++++++--------- .../tostring/GridToStringThreadLocal.java | 66 ++ .../util/tostring/SBLimitedLength.java | 20 - .../tostring/GridToStringBuilderSelfTest.java | 247 +----- 4 files changed, 439 insertions(+), 600 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java index 86daf7cc7b21e..77c333daa774c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java @@ -28,16 +28,19 @@ import java.util.Collection; import java.util.Collections; import java.util.EventListener; -import java.util.IdentityHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -47,9 +50,6 @@ /** * Provides auto-generation framework for {@code toString()} output. *

- * In case of recursion, repeatable objects will be shown as "ClassName@hash". - * But fields will be printed only for the first entry to prevent recursion. - *

* Default exclusion policy (can be overridden with {@link GridToStringInclude} * annotation): *

    @@ -83,9 +83,6 @@ *
*/ public class GridToStringBuilder { - /** */ - private static final Object[] EMPTY_ARRAY = new Object[0]; - /** */ private static final Map classCache = new ConcurrentHashMap<>(); @@ -97,30 +94,25 @@ public class GridToStringBuilder { private static final int COLLECTION_LIMIT = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100); - /** Every thread has its own string builder. */ - private static ThreadLocal threadLocSB = new ThreadLocal() { - @Override protected SBLimitedLength initialValue() { - SBLimitedLength sb = new SBLimitedLength(256); + /** */ + private static ThreadLocal> threadCache = new ThreadLocal>() { + @Override protected Queue initialValue() { + Queue queue = new LinkedList<>(); - sb.initLimit(new SBLengthLimit()); + queue.offer(new GridToStringThreadLocal()); - return sb; + return queue; } }; - /** - * Contains objects currently printing in the string builder. - *

- * Since {@code toString()} methods can be chain-called from the same thread we - * have to keep a map of this objects pointed to the position of previous occurrence - * and remove/add them in each {@code toString()} apply. - */ - private static ThreadLocal> savedObjects = new ThreadLocal>() { - @Override protected IdentityHashMap initialValue() { - return new IdentityHashMap<>(); + /** */ + private static ThreadLocal threadCurLen = new ThreadLocal() { + @Override protected SBLengthLimit initialValue() { + return new SBLengthLimit(); } }; + /** * Produces auto-generated output of string presentation for given object and its declaration class. * @@ -269,9 +261,18 @@ public static String toString(Class cls, T obj, assert name3 != null; assert name4 != null; - Object[] addNames = new Object[5]; - Object[] addVals = new Object[5]; - boolean[] addSens = new boolean[5]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name0; addVals[0] = val0; @@ -289,16 +290,20 @@ public static String toString(Class cls, T obj, addVals[4] = val4; addSens[4] = sens4; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 5); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 5); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -344,9 +349,18 @@ public static String toString(Class cls, T obj, assert name4 != null; assert name5 != null; - Object[] addNames = new Object[6]; - Object[] addVals = new Object[6]; - boolean[] addSens = new boolean[6]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name0; addVals[0] = val0; @@ -367,16 +381,20 @@ public static String toString(Class cls, T obj, addVals[5] = val5; addSens[5] = sens5; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 6); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 6); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -427,9 +445,18 @@ public static String toString(Class cls, T obj, assert name5 != null; assert name6 != null; - Object[] addNames = new Object[7]; - Object[] addVals = new Object[7]; - boolean[] addSens = new boolean[7]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name0; addVals[0] = val0; @@ -453,16 +480,20 @@ public static String toString(Class cls, T obj, addVals[6] = val6; addSens[6] = sens6; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 7); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 7); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -526,9 +557,18 @@ public static String toString(Class cls, T obj, assert name2 != null; assert name3 != null; - Object[] addNames = new Object[4]; - Object[] addVals = new Object[4]; - boolean[] addSens = new boolean[4]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name0; addVals[0] = val0; @@ -543,16 +583,20 @@ public static String toString(Class cls, T obj, addVals[3] = val3; addSens[3] = sens3; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 4); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 4); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -608,9 +652,18 @@ public static String toString(Class cls, T obj, assert name1 != null; assert name2 != null; - Object[] addNames = new Object[3]; - Object[] addVals = new Object[3]; - boolean[] addSens = new boolean[3]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name0; addVals[0] = val0; @@ -622,16 +675,20 @@ public static String toString(Class cls, T obj, addVals[2] = val2; addSens[2] = sens2; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 3); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 3); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -675,9 +732,18 @@ public static String toString(Class cls, T obj, assert name0 != null; assert name1 != null; - Object[] addNames = new Object[2]; - Object[] addVals = new Object[2]; - boolean[] addSens = new boolean[2]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name0; addVals[0] = val0; @@ -686,16 +752,20 @@ public static String toString(Class cls, T obj, addVals[1] = val1; addSens[1] = sens1; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 2); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 2); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -729,24 +799,37 @@ public static String toString(Class cls, T obj, String name, @Nullable Ob assert obj != null; assert name != null; - Object[] addNames = new Object[1]; - Object[] addVals = new Object[1]; - boolean[] addSens = new boolean[1]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] addNames = tmp.getAdditionalNames(); + Object[] addVals = tmp.getAdditionalValues(); + boolean[] addSens = tmp.getAdditionalSensitives(); addNames[0] = name; addVals[0] = val; addSens[0] = sens; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 1); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 1); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -762,16 +845,30 @@ public static String toString(Class cls, T obj) { assert cls != null; assert obj != null; - SBLimitedLength sb = threadLocSB.get(); + Queue queue = threadCache.get(); + + assert queue != null; - boolean newStr = sb.length() == 0; + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + SBLengthLimit lenLim = threadCurLen.get(); + + boolean newStr = false; try { - return toStringImpl(cls, sb, obj, EMPTY_ARRAY, EMPTY_ARRAY, null, 0); + newStr = lenLim.length() == 0; + + return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, tmp.getAdditionalNames(), + tmp.getAdditionalValues(), null, 0); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -789,164 +886,68 @@ public static String toString(Class cls, T obj, String parent) { } /** - * Print value with length limitation. - * + * Print value with length limitation * @param buf buffer to print to. * @param val value to print, can be {@code null}. */ private static void toString(SBLimitedLength buf, Object val) { - toString(buf, null, val); - } - - /** - * Print value with length limitation. - * - * @param buf buffer to print to. - * @param cls value class. - * @param val value to print. - */ - @SuppressWarnings({"unchecked"}) - private static void toString(SBLimitedLength buf, Class cls, Object val) { - if (val == null) { + if (val == null) buf.a("null"); - - return; - } - - if (cls == null) - cls = val.getClass(); - - if (cls.isPrimitive()) { - buf.a(val); - - return; - } - - IdentityHashMap svdObjs = savedObjects.get(); - - if (handleRecursion(buf, val, svdObjs)) - return; - - svdObjs.put(val, buf.length()); - - try { - if (cls.isArray()) - addArray(buf, cls, val); - else if (val instanceof Collection) - addCollection(buf, (Collection) val); - else if (val instanceof Map) - addMap(buf, (Map) val); - else - buf.a(val); - } - finally { - svdObjs.remove(val); - } - } - - /** - * Writes array to buffer. - * - * @param buf String builder buffer. - * @param arrType Type of the array. - * @param obj Array object. - */ - private static void addArray(SBLimitedLength buf, Class arrType, Object obj) { - if (arrType.getComponentType().isPrimitive()) { - buf.a(arrayToString(arrType, obj)); - - return; - } - - Object[] arr = (Object[]) obj; - - buf.a(arrType.getSimpleName()).a(" ["); - - for (int i = 0; i < arr.length; i++) { - toString(buf, arr[i]); - - if (i == COLLECTION_LIMIT - 1 || i == arr.length - 1) - break; - - buf.a(", "); - } - - handleOverflow(buf, arr.length); - - buf.a(']'); + else + toString(buf, val.getClass(), val); } /** - * Writes collection to buffer. - * - * @param buf String builder buffer. - * @param col Collection object. + * Print value with length limitation + * @param buf buffer to print to. + * @param valClass value class. + * @param val value to print */ - private static void addCollection(SBLimitedLength buf, Collection col) { - buf.a(col.getClass().getSimpleName()).a(" ["); - - int cnt = 0; - - for (Object obj : col) { - toString(buf, obj); + private static void toString(SBLimitedLength buf, Class valClass, Object val) { + if (valClass.isArray()) + buf.a(arrayToString(valClass, val)); + else { + int overflow = 0; + char bracket = ' '; - if (++cnt == COLLECTION_LIMIT || cnt == col.size()) - break; + if (val instanceof Collection && ((Collection)val).size() > COLLECTION_LIMIT) { + overflow = ((Collection)val).size() - COLLECTION_LIMIT; + bracket = ']'; + val = F.retain((Collection) val, true, COLLECTION_LIMIT); + } + else if (val instanceof Map && ((Map)val).size() > COLLECTION_LIMIT) { + Map tmp = U.newHashMap(COLLECTION_LIMIT); - buf.a(", "); - } + overflow = ((Map)val).size() - COLLECTION_LIMIT; - handleOverflow(buf, col.size()); + bracket= '}'; - buf.a(']'); - } - - /** - * Writes map to buffer. - * - * @param buf String builder buffer. - * @param map Map object. - */ - private static void addMap(SBLimitedLength buf, Map map) { - buf.a(map.getClass().getSimpleName()).a(" {"); + int cntr = 0; - int cnt = 0; + for (Object o : ((Map)val).entrySet()) { + Map.Entry e = (Map.Entry)o; - for (Map.Entry e : map.entrySet()) { - toString(buf, e.getKey()); + tmp.put(e.getKey(), e.getValue()); - buf.a('='); + if (++cntr >= COLLECTION_LIMIT) + break; + } - toString(buf, e.getValue()); + val = tmp; + } - if (++cnt == COLLECTION_LIMIT || cnt == map.size()) - break; + buf.a(val); - buf.a(", "); + if (overflow > 0) { + buf.d(buf.length() - 1); + buf.a("... and ").a(overflow).a(" more").a(bracket); + } } - - handleOverflow(buf, map.size()); - - buf.a('}'); - } - - /** - * Writes overflow message to buffer if needed. - * - * @param buf String builder buffer. - * @param size Size to compare with limit. - */ - private static void handleOverflow(SBLimitedLength buf, int size) { - int overflow = size - COLLECTION_LIMIT; - - if (overflow > 0) - buf.a("... and ").a(overflow).a(" more"); } /** * Creates an uniformed string presentation for the given object. * - * @param Type of object. * @param cls Class of the object. * @param buf String builder buffer. * @param obj Object for which to get string presentation. @@ -955,7 +956,9 @@ private static void handleOverflow(SBLimitedLength buf, int size) { * @param addSens Sensitive flag of values or {@code null} if all values are not sensitive. * @param addLen How many additional values will be included. * @return String presentation of the given object. + * @param Type of object. */ + @SuppressWarnings({"unchecked"}) private static String toStringImpl( Class cls, SBLimitedLength buf, @@ -972,55 +975,13 @@ private static String toStringImpl( assert addNames.length == addVals.length; assert addLen <= addNames.length; - boolean newStr = buf.length() == 0; - - IdentityHashMap svdObjs = savedObjects.get(); - - if (newStr) - svdObjs.put(obj, buf.length()); - - try { - String s = toStringImpl0(cls, buf, obj, addNames, addVals, addSens, addLen); - - if (newStr) - return s; - - return ""; - } - finally { - if (newStr) - svdObjs.remove(obj); - } - } - - /** - * Creates an uniformed string presentation for the given object. - * - * @param cls Class of the object. - * @param buf String builder buffer. - * @param obj Object for which to get string presentation. - * @param addNames Names of additional values to be included. - * @param addVals Additional values to be included. - * @param addSens Sensitive flag of values or {@code null} if all values are not sensitive. - * @param addLen How many additional values will be included. - * @return String presentation of the given object. - * @param Type of object. - */ - @SuppressWarnings({"unchecked"}) - private static String toStringImpl0( - Class cls, - SBLimitedLength buf, - T obj, - Object[] addNames, - Object[] addVals, - @Nullable boolean[] addSens, - int addLen - ) { try { GridToStringClassDescriptor cd = getClassDescriptor(cls); assert cd != null; + buf.setLength(0); + buf.a(cd.getSimpleClassName()).a(" ["); boolean first = true; @@ -1183,24 +1144,37 @@ else if (arrType.equals(char[].class)) { public static String toString(String str, String name, @Nullable Object val, boolean sens) { assert name != null; - Object[] propNames = new Object[1]; - Object[] propVals = new Object[1]; - boolean[] propSens = new boolean[1]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name; propVals[0] = val; propSens[0] = sens; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 1); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 1); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1237,9 +1211,18 @@ public static String toString(String str, assert name0 != null; assert name1 != null; - Object[] propNames = new Object[2]; - Object[] propVals = new Object[2]; - boolean[] propSens = new boolean[2]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name0; propVals[0] = val0; @@ -1248,16 +1231,20 @@ public static String toString(String str, propVals[1] = val1; propSens[1] = sens1; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 2); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 2); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1284,9 +1271,18 @@ public static String toString(String str, assert name1 != null; assert name2 != null; - Object[] propNames = new Object[3]; - Object[] propVals = new Object[3]; - boolean[] propSens = new boolean[3]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name0; propVals[0] = val0; @@ -1298,16 +1294,20 @@ public static String toString(String str, propVals[2] = val2; propSens[2] = sens2; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 3); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 3); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1339,9 +1339,18 @@ public static String toString(String str, assert name2 != null; assert name3 != null; - Object[] propNames = new Object[4]; - Object[] propVals = new Object[4]; - boolean[] propSens = new boolean[4]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name0; propVals[0] = val0; @@ -1356,16 +1365,20 @@ public static String toString(String str, propVals[3] = val3; propSens[3] = sens3; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 4); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 4); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1402,9 +1415,18 @@ public static String toString(String str, assert name3 != null; assert name4 != null; - Object[] propNames = new Object[5]; - Object[] propVals = new Object[5]; - boolean[] propSens = new boolean[5]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name0; propVals[0] = val0; @@ -1422,16 +1444,20 @@ public static String toString(String str, propVals[4] = val4; propSens[4] = sens4; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 5); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 5); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1473,9 +1499,18 @@ public static String toString(String str, assert name4 != null; assert name5 != null; - Object[] propNames = new Object[6]; - Object[] propVals = new Object[6]; - boolean[] propSens = new boolean[6]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name0; propVals[0] = val0; @@ -1496,16 +1531,20 @@ public static String toString(String str, propVals[5] = val5; propSens[5] = sens5; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 6); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 6); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1552,9 +1591,18 @@ public static String toString(String str, assert name5 != null; assert name6 != null; - Object[] propNames = new Object[7]; - Object[] propVals = new Object[7]; - boolean[] propSens = new boolean[7]; + Queue queue = threadCache.get(); + + assert queue != null; + + // Since string() methods can be chain-called from the same thread we + // have to keep a list of thread-local objects and remove/add them + // in each string() apply. + GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove(); + + Object[] propNames = tmp.getAdditionalNames(); + Object[] propVals = tmp.getAdditionalValues(); + boolean[] propSens = tmp.getAdditionalSensitives(); propNames[0] = name0; propVals[0] = val0; @@ -1578,16 +1626,20 @@ public static String toString(String str, propVals[6] = val6; propSens[6] = sens6; - SBLimitedLength sb = threadLocSB.get(); + SBLengthLimit lenLim = threadCurLen.get(); - boolean newStr = sb.length() == 0; + boolean newStr = false; try { - return toStringImpl(str, sb, propNames, propVals, propSens, 7); + newStr = lenLim.length() == 0; + + return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 7); } finally { + queue.offer(tmp); + if (newStr) - sb.reset(); + lenLim.reset(); } } @@ -1789,54 +1841,4 @@ public static String compact(@NotNull Collection col) { return sb.toString(); } - - /** - * Checks that object is already saved. - * In positive case this method inserts hash to the saved object entry (if needed) and name@hash for current entry. - * Further toString operations are not needed for current object. - * - * @param buf String builder buffer. - * @param obj Object. - * @param svdObjs Map with saved objects to handle recursion. - * @return {@code True} if object is already saved and name@hash was added to buffer. - * {@code False} if it wasn't saved previously and it should be saved. - */ - private static boolean handleRecursion(SBLimitedLength buf, Object obj, IdentityHashMap svdObjs) { - Integer pos = svdObjs.get(obj); - - if (pos == null) - return false; - - String name = obj.getClass().getSimpleName(); - String hash = '@' + Integer.toHexString(System.identityHashCode(obj)); - String savedName = name + hash; - - if (!buf.isOverflowed() && buf.impl().indexOf(savedName, pos) != pos) { - buf.i(pos + name.length(), hash); - - incValues(svdObjs, obj, hash.length()); - } - - buf.a(savedName); - - return true; - } - - /** - * Increment positions of already presented objects afterward given object. - * - * @param svdObjs Map with objects already presented in the buffer. - * @param obj Object. - * @param hashLen Length of the object's hash. - */ - private static void incValues(IdentityHashMap svdObjs, Object obj, int hashLen) { - Integer baseline = svdObjs.get(obj); - - for (IdentityHashMap.Entry entry : svdObjs.entrySet()) { - Integer pos = entry.getValue(); - - if (pos > baseline) - entry.setValue(pos + hashLen); - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java new file mode 100644 index 0000000000000..2f62727b17802 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringThreadLocal.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.util.tostring; + +/** + * Helper wrapper containing StringBuilder and additional values. Stored as a thread-local variable. + */ +class GridToStringThreadLocal { + /** */ + private SBLimitedLength sb = new SBLimitedLength(256); + + /** */ + private Object[] addNames = new Object[7]; + + /** */ + private Object[] addVals = new Object[7]; + + /** */ + private boolean[] addSens = new boolean[7]; + + /** + * @param len Length limit. + * @return String builder. + */ + SBLimitedLength getStringBuilder(SBLengthLimit len) { + sb.initLimit(len); + + return sb; + } + + /** + * @return Additional names. + */ + Object[] getAdditionalNames() { + return addNames; + } + + /** + * @return Additional values. + */ + Object[] getAdditionalValues() { + return addVals; + } + + /** + * @return Additional values. + */ + boolean[] getAdditionalSensitives() { + return addSens; + } +} \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java index b524b3d9ad39c..c47cf40c26ef7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java @@ -50,19 +50,6 @@ void initLimit(SBLengthLimit lenLimit) { tail.reset(); } - /** - * Resets buffer. - */ - public void reset() { - super.setLength(0); - - lenLimit.reset(); - - if (tail != null) - tail.reset(); - } - - /** * @return tail string builder. */ @@ -305,11 +292,4 @@ private GridStringBuilder onWrite(int lenBeforeWrite) { return res.toString(); } } - - /** - * @return {@code True} - if buffer limit is reached. - */ - public boolean isOverflowed() { - return lenLimit.overflowed(this); - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java index eff3349f37e44..4ac05fb64f398 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java @@ -20,25 +20,19 @@ import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.CyclicBarrier; import java.util.concurrent.locks.ReadWriteLock; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; -import org.apache.ignite.internal.IgniteInternalFuture; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_COLLECTION_LIMIT; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_MAX_LENGTH; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.testframework.junits.common.GridCommonTest; -import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_COLLECTION_LIMIT; -import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_MAX_LENGTH; - /** * Tests for {@link GridToStringBuilder}. */ @@ -78,164 +72,32 @@ public void testToStringWithAdditions() throws Exception { /** * @throws Exception If failed. */ - public void testToStringCheckSimpleListRecursionPrevention() throws Exception { + public void testToStringCheckSimpleRecursionPrevention() throws Exception { ArrayList list1 = new ArrayList<>(); ArrayList list2 = new ArrayList<>(); list2.add(list1); list1.add(list2); - info(GridToStringBuilder.toString(ArrayList.class, list1)); - info(GridToStringBuilder.toString(ArrayList.class, list2)); - } - - /** - * @throws Exception If failed. - */ - public void testToStringCheckSimpleMapRecursionPrevention() throws Exception { - HashMap map1 = new HashMap<>(); - HashMap map2 = new HashMap<>(); - - map1.put("2", map2); - map2.put("1", map1); - info(GridToStringBuilder.toString(HashMap.class, map1)); - info(GridToStringBuilder.toString(HashMap.class, map2)); + GridToStringBuilder.toString(ArrayList.class, list1); + GridToStringBuilder.toString(ArrayList.class, list2); } /** * @throws Exception If failed. */ - public void testToStringCheckListAdvancedRecursionPrevention() throws Exception { + public void testToStringCheckAdvancedRecursionPrevention() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-602"); + ArrayList list1 = new ArrayList<>(); ArrayList list2 = new ArrayList<>(); list2.add(list1); list1.add(list2); - info(GridToStringBuilder.toString(ArrayList.class, list1, "name", list2)); - info(GridToStringBuilder.toString(ArrayList.class, list2, "name", list1)); - } - - /** - * @throws Exception If failed. - */ - public void testToStringCheckMapAdvancedRecursionPrevention() throws Exception { - HashMap map1 = new HashMap<>(); - HashMap map2 = new HashMap<>(); - - map1.put("2", map2); - map2.put("1", map1); - - info(GridToStringBuilder.toString(HashMap.class, map1, "name", map2)); - info(GridToStringBuilder.toString(HashMap.class, map2, "name", map1)); - } - - /** - * @throws Exception If failed. - */ - public void testToStringCheckObjectRecursionPrevention() throws Exception { - Node n1 = new Node(); - Node n2 = new Node(); - Node n3 = new Node(); - Node n4 = new Node(); - - n1.name = "n1"; - n2.name = "n2"; - n3.name = "n3"; - n4.name = "n4"; - - n1.next = n2; - n2.next = n3; - n3.next = n4; - n4.next = n3; - - n1.nodes = new Node[4]; - n2.nodes = n1.nodes; - n3.nodes = n1.nodes; - n4.nodes = n1.nodes; - - n1.nodes[0] = n1; - n1.nodes[1] = n2; - n1.nodes[2] = n3; - n1.nodes[3] = n4; - - String expN1 = n1.toString(); - String expN2 = n2.toString(); - String expN3 = n3.toString(); - String expN4 = n4.toString(); - - info(expN1); - info(expN2); - info(expN3); - info(expN4); - info(GridToStringBuilder.toString("Test", "Appended vals", n1)); - - CyclicBarrier bar = new CyclicBarrier(4); - - IgniteInternalFuture fut1 = GridTestUtils.runAsync(new BarrierCallable(bar, n1, expN1)); - IgniteInternalFuture fut2 = GridTestUtils.runAsync(new BarrierCallable(bar, n2, expN2)); - IgniteInternalFuture fut3 = GridTestUtils.runAsync(new BarrierCallable(bar, n3, expN3)); - IgniteInternalFuture fut4 = GridTestUtils.runAsync(new BarrierCallable(bar, n4, expN4)); - - fut1.get(3_000); - fut2.get(3_000); - fut3.get(3_000); - fut4.get(3_000); - } - - /** - * Test class. - */ - private static class Node { - /** */ - @GridToStringInclude - String name; - - /** */ - @GridToStringInclude - Node next; - - /** */ - @GridToStringInclude - Node[] nodes; - - /** {@inheritDoc} */ - @Override public String toString() { - return GridToStringBuilder.toString(Node.class, this); - } - } - - /** - * Test class. - */ - private static class BarrierCallable implements Callable { - /** */ - CyclicBarrier bar; - - /** */ - Object obj; - - /** Expected value of {@code toString()} method. */ - String exp; - - /** */ - private BarrierCallable(CyclicBarrier bar, Object obj, String exp) { - this.bar = bar; - this.obj = obj; - this.exp = exp; - } - - /** {@inheritDoc} */ - @Override public String call() throws Exception { - for (int i = 0; i < 10; i++) { - bar.await(); - - assertEquals(exp, obj.toString()); - } - - return null; - } + GridToStringBuilder.toString(ArrayList.class, list1, "name", list2); + GridToStringBuilder.toString(ArrayList.class, list2, "name", list1); } /** @@ -275,33 +137,6 @@ private void testArr(V v, int limit) throws Exception { Arrays.fill(arrOf, v); T[] arr = Arrays.copyOf(arrOf, limit); - checkArrayOverflow(arrOf, arr, limit); - } - - /** - * Test array print. - * - * @throws Exception if failed. - */ - public void testArrLimitWithRecursion() throws Exception { - int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100); - - ArrayList[] arrOf = new ArrayList[limit + 1]; - Arrays.fill(arrOf, new ArrayList()); - ArrayList[] arr = Arrays.copyOf(arrOf, limit); - - arrOf[0].add(arrOf); - arr[0].add(arr); - - checkArrayOverflow(arrOf, arr, limit); - } - - /** - * @param arrOf Array. - * @param arr Array copy. - * @param limit Array limit. - */ - private void checkArrayOverflow(Object[] arrOf, Object[] arr, int limit) { String arrStr = GridToStringBuilder.arrayToString(arr.getClass(), arr); String arrOfStr = GridToStringBuilder.arrayToString(arrOf.getClass(), arrOf); @@ -309,11 +144,7 @@ private void checkArrayOverflow(Object[] arrOf, Object[] arr, int limit) { StringBuilder resultSB = new StringBuilder(arrStr); resultSB.deleteCharAt(resultSB.length()-1); resultSB.append("... and ").append(arrOf.length - limit).append(" more]"); - - arrStr = resultSB.toString(); - - info(arrOfStr); - info(arrStr); + arrStr = resultSB.toString(); assertTrue("Collection limit error in array of type " + arrOf.getClass().getName() + " error, normal arr: <" + arrStr + ">, overflowed arr: <" + arrOfStr + ">", arrStr.equals(arrOfStr)); @@ -397,58 +228,18 @@ public void testToStringCollectionLimits() throws Exception { strMap.put("k" + i, "v"); strList.add("e"); } + String testClassStr = GridToStringBuilder.toString(TestClass1.class, testClass); - checkColAndMap(testClass); - } - - /** - * @throws Exception If failed. - */ - public void testToStringColAndMapLimitWithRecursion() throws Exception { - int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100); - Map strMap = new TreeMap<>(); - List strList = new ArrayList<>(limit+1); - - TestClass1 testClass = new TestClass1(); - testClass.strMap = strMap; - testClass.strListIncl = strList; - - Map m = new TreeMap(); - m.put("m", strMap); - - List l = new ArrayList(); - l.add(strList); - - strMap.put("k0", m); - strList.add(l); - - for (int i = 1; i < limit; i++) { - strMap.put("k" + i, "v"); - strList.add("e"); - } - - checkColAndMap(testClass); - } - - /** - * @param testCls Class with collection and map included in toString(). - */ - private void checkColAndMap(TestClass1 testCls) { - String testClsStr = GridToStringBuilder.toString(TestClass1.class, testCls); - - testCls.strMap.put("kz", "v"); // important to add last element in TreeMap here - testCls.strListIncl.add("e"); + strMap.put("kz", "v"); // important to add last element in TreeMap here + strList.add("e"); - String testClsStrOf = GridToStringBuilder.toString(TestClass1.class, testCls); + String testClassStrOf = GridToStringBuilder.toString(TestClass1.class, testClass); - String testClsStrOfR = testClsStrOf.replaceAll("... and 1 more",""); + String testClassStrOfR = testClassStrOf.replaceAll("... and 1 more",""); - info(testClsStr); - info(testClsStrOf); - info(testClsStrOfR); + assertTrue("Collection limit error in Map or List, normal: <" + testClassStr + ">, overflowed: <" + +"testClassStrOf", testClassStr.length() == testClassStrOfR.length()); - assertTrue("Collection limit error in Map or List, normal: <" + testClsStr + ">, overflowed: <" - + testClsStrOf + ">", testClsStr.length() == testClsStrOfR.length()); } /** @@ -611,4 +402,4 @@ public TestClass2(String str) { this.str = str; } } -} +} \ No newline at end of file From 4b4192121173b38f79f1898448e08170d9c0917c Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Wed, 21 Nov 2018 20:35:23 +0300 Subject: [PATCH 171/403] IGNITE-10339 Fixed cluster connection when using validate indexes from control.sh - Fixes #5458. Signed-off-by: Alexey Goncharuk (cherry picked from commit 5fc9a3fc1ef5c486e2656661e2e811b235b6543f) --- .../internal/commandline/CommandHandler.java | 48 +++++++++---------- .../verify/VisorValidateIndexesJobResult.java | 9 ++++ .../verify/VisorValidateIndexesTaskArg.java | 23 +++++++-- .../verify/VisorValidateIndexesTask.java | 29 +++++++++++ .../util/GridCommandHandlerIndexingTest.java | 2 +- 5 files changed, 83 insertions(+), 28 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index e37da813c6ede..12649d78d50b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -813,39 +813,41 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException { VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg( cacheArgs.caches(), + cacheArgs.nodeId() != null ? Collections.singleton(cacheArgs.nodeId()) : null, cacheArgs.checkFirst(), cacheArgs.checkThrough() ); - UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId(); - VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode( - client, VALIDATE_INDEXES_TASK, taskArg, nodeId); + client, VALIDATE_INDEXES_TASK, taskArg, null); if (!F.isEmpty(taskRes.exceptions())) { log("Index validation failed on nodes:"); for (Map.Entry e : taskRes.exceptions().entrySet()) { - log("Node ID = " + e.getKey()); + log(i("Node ID = " + e.getKey())); - log("Exception message:"); - log(e.getValue().getMessage()); + log(i("Exception message:")); + log(i(e.getValue().getMessage(), 2)); nl(); } } + boolean errors = false; + for (Map.Entry nodeEntry : taskRes.results().entrySet()) { - boolean errors = false; + if (!nodeEntry.getValue().hasIssues()) + continue; - log("validate_indexes result on node " + nodeEntry.getKey() + ":"); + errors = true; + + log("Index issues found on node " + nodeEntry.getKey() + ":"); Collection integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures(); if (!integrityCheckFailures.isEmpty()) { - errors = true; - for (IndexIntegrityCheckIssue is : integrityCheckFailures) - log("\t" + is.toString()); + log(i(is.toString())); } Map partRes = nodeEntry.getValue().partitionResult(); @@ -854,12 +856,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t ValidateIndexesPartitionResult res = e.getValue(); if (!res.issues().isEmpty()) { - errors = true; - - log("\t" + e.getKey().toString() + " " + e.getValue().toString()); + log(i(e.getKey().toString() + " " + e.getValue().toString())); for (IndexValidationIssue is : res.issues()) - log("\t\t" + is.toString()); + log(i(is.toString(), 2)); } } @@ -869,20 +869,20 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t ValidateIndexesPartitionResult res = e.getValue(); if (!res.issues().isEmpty()) { - errors = true; - - log("\tSQL Index " + e.getKey() + " " + e.getValue().toString()); + log(i("SQL Index " + e.getKey() + " " + e.getValue().toString())); for (IndexValidationIssue is : res.issues()) - log("\t\t" + is.toString()); + log(i(is.toString(),2)); } } - - if (!errors) - log("no issues found.\n"); - else - log("issues found (listed above).\n"); } + + if (!errors) + log("no issues found."); + else + log("issues found (listed above)."); + + nl(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java index f84fc1a259027..6d7f7652518cd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java @@ -92,6 +92,15 @@ public Collection integrityCheckFailures() { return integrityCheckFailures == null ? Collections.emptyList() : integrityCheckFailures; } + /** + * @return {@code true} If any indexes issues found on node, otherwise returns {@code false}. + */ + public boolean hasIssues() { + return (integrityCheckFailures != null && !integrityCheckFailures.isEmpty()) || + (partRes != null && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty())) || + (idxRes != null && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty())); + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeMap(out, partRes); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java index aa49977c7a6fa..6dbd961028e69 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java @@ -21,12 +21,14 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.Set; +import java.util.UUID; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; + /** - * Arguments for task {@link VisorIdleVerifyTask} + * */ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject { /** */ @@ -41,6 +43,9 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject { /** Check through K element (skip K-1, check Kth). */ private int checkThrough; + /** Nodes on which task will run. */ + private Set nodes; + /** * Default constructor. */ @@ -51,10 +56,11 @@ public VisorValidateIndexesTaskArg() { /** * @param caches Caches. */ - public VisorValidateIndexesTaskArg(Set caches, int checkFirst, int checkThrough) { + public VisorValidateIndexesTaskArg(Set caches, Set nodes, int checkFirst, int checkThrough) { this.caches = caches; this.checkFirst = checkFirst; this.checkThrough = checkThrough; + this.nodes = nodes; } @@ -65,6 +71,13 @@ public Set getCaches() { return caches; } + /** + * @return Nodes on which task will run. If {@code null}, task will run on all server nodes. + */ + public Set getNodes() { + return nodes; + } + /** * @return checkFirst. */ @@ -84,6 +97,7 @@ public int getCheckThrough() { U.writeCollection(out, caches); out.writeInt(checkFirst); out.writeInt(checkThrough); + U.writeCollection(out, nodes); } /** {@inheritDoc} */ @@ -98,11 +112,14 @@ public int getCheckThrough() { checkFirst = -1; checkThrough = -1; } + + if (protoVer > V2) + nodes = U.readSet(in); } /** {@inheritDoc} */ @Override public byte getProtocolVersion() { - return V2; + return V3; } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java index abb7f7ee55ec1..922c53e15ed97 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java @@ -17,16 +17,22 @@ package org.apache.ignite.internal.visor.verify; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; +import org.apache.ignite.internal.visor.VisorTaskArgument; import org.jetbrains.annotations.Nullable; /** @@ -58,6 +64,29 @@ public class VisorValidateIndexesTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { + Collection srvNodes = ignite.cluster().forServers().nodes(); + Collection ret = new ArrayList<>(srvNodes.size()); + + VisorValidateIndexesTaskArg taskArg = arg.getArgument(); + + Set nodeIds = taskArg.getNodes() != null ? new HashSet<>(taskArg.getNodes()) : null; + + if (nodeIds == null) { + for (ClusterNode node : srvNodes) + ret.add(node.id()); + } + else { + for (ClusterNode node : srvNodes) { + if (nodeIds.contains(node.id())) + ret.add(node.id()); + } + } + + return ret; + } + /** * */ diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java index c7693d214b0ac..dc5011ffe170b 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java @@ -113,7 +113,7 @@ public void testBrokenSqlIndexShouldFailValidation() throws Exception { } /** - * Tests that missing rows in H2 indexes are detected. + * Tests that corrupted pages in the index partition are detected. */ public void testCorruptedIndexPartitionShouldFailValidation() throws Exception { Ignite ignite = prepareGridForTest(); From d87f201b7c6a6823eb96f48c2c5d00661ad018b9 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Fri, 23 Nov 2018 19:36:00 +0300 Subject: [PATCH 172/403] IGNITE-10339 Skip index partition file integrity check for in-memory caches - Fixes #5475. Signed-off-by: Alexey Goncharuk (cherry picked from commit 5b656e2f3b2157200d650923e5aca800c4d61635) --- .../ignite/internal/commandline/CommandHandler.java | 6 ++++-- .../internal/visor/verify/ValidateIndexesClosure.java | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 12649d78d50b2..899f7ef845a05 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -821,7 +821,11 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode( client, VALIDATE_INDEXES_TASK, taskArg, null); + boolean errors = false; + if (!F.isEmpty(taskRes.exceptions())) { + errors = true; + log("Index validation failed on nodes:"); for (Map.Entry e : taskRes.exceptions().entrySet()) { @@ -833,8 +837,6 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t } } - boolean errors = false; - for (Map.Entry nodeEntry : taskRes.results().entrySet()) { if (!nodeEntry.getValue().hasIssues()) continue; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java index ec02c25c3ca36..b6909e3191e7f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java @@ -260,7 +260,8 @@ private VisorValidateIndexesJobResult call0() throws Exception { Map partRes = fut.get(); - partResults.putAll(partRes); + if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty())) + partResults.putAll(partRes); } for (; curIdx < procIdxFutures.size(); curIdx++) { @@ -268,7 +269,8 @@ private VisorValidateIndexesJobResult call0() throws Exception { Map idxRes = fut.get(); - idxResults.putAll(idxRes); + if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty())) + idxResults.putAll(idxRes); } log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " @@ -296,7 +298,7 @@ private Map integrityCheckIndexesPartitions(S for (Integer grpId: grpIds) { final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId); - if (grpCtx == null) { + if (grpCtx == null || !grpCtx.persistenceEnabled()) { integrityCheckedIndexes.incrementAndGet(); continue; From 2e32dbc698639dc0771a3211d0c56eed26e9ed9c Mon Sep 17 00:00:00 2001 From: Dmitry Melnichuk Date: Mon, 26 Nov 2018 17:14:59 +0300 Subject: [PATCH 173/403] IGNITE-10358: Added collections data type specification for python thin client This closes #5470 (cherry picked from commit 04fae6d2cbbd83b15a43cf17da9011c0eccec41b) --- .../docs/source/pyignite.datatypes.base.rst | 7 ++ .../python/docs/source/pyignite.datatypes.rst | 1 + .../python/pyignite/datatypes/base.py | 24 ++++++ .../python/pyignite/datatypes/complex.py | 23 +++--- .../python/pyignite/datatypes/internal.py | 19 ++++- .../python/pyignite/datatypes/null_object.py | 3 +- .../python/pyignite/datatypes/primitive.py | 3 +- .../pyignite/datatypes/primitive_arrays.py | 3 +- .../pyignite/datatypes/primitive_objects.py | 3 +- .../python/pyignite/datatypes/standard.py | 9 ++- modules/platforms/python/pyignite/utils.py | 6 +- modules/platforms/python/setup.py | 2 +- .../platforms/python/tests/test_key_value.py | 75 ++++++++++++++++++- 13 files changed, 149 insertions(+), 29 deletions(-) create mode 100644 modules/platforms/python/docs/source/pyignite.datatypes.base.rst create mode 100644 modules/platforms/python/pyignite/datatypes/base.py diff --git a/modules/platforms/python/docs/source/pyignite.datatypes.base.rst b/modules/platforms/python/docs/source/pyignite.datatypes.base.rst new file mode 100644 index 0000000000000..849a02817a04a --- /dev/null +++ b/modules/platforms/python/docs/source/pyignite.datatypes.base.rst @@ -0,0 +1,7 @@ +pyignite.datatypes.base module +============================== + +.. automodule:: pyignite.datatypes.base + :members: + :undoc-members: + :show-inheritance: diff --git a/modules/platforms/python/docs/source/pyignite.datatypes.rst b/modules/platforms/python/docs/source/pyignite.datatypes.rst index 77e7183bd0a91..d72f84432d835 100644 --- a/modules/platforms/python/docs/source/pyignite.datatypes.rst +++ b/modules/platforms/python/docs/source/pyignite.datatypes.rst @@ -11,6 +11,7 @@ Submodules .. toctree:: + pyignite.datatypes.base pyignite.datatypes.binary pyignite.datatypes.cache_config pyignite.datatypes.cache_properties diff --git a/modules/platforms/python/pyignite/datatypes/base.py b/modules/platforms/python/pyignite/datatypes/base.py new file mode 100644 index 0000000000000..a0522c0ac3e45 --- /dev/null +++ b/modules/platforms/python/pyignite/datatypes/base.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC + + +class IgniteDataType(ABC): + """ + This is a base class for all Ignite data types, a.k.a. parser/constructor + classes, both object and payload varieties. + """ + pass diff --git a/modules/platforms/python/pyignite/datatypes/complex.py b/modules/platforms/python/pyignite/datatypes/complex.py index 9a5664c97c74e..87e5130bf3bdc 100644 --- a/modules/platforms/python/pyignite/datatypes/complex.py +++ b/modules/platforms/python/pyignite/datatypes/complex.py @@ -20,7 +20,8 @@ from pyignite.constants import * from pyignite.exceptions import ParseError from pyignite.utils import entity_id, hashcode, is_hinted -from .internal import AnyDataObject +from .base import IgniteDataType +from .internal import AnyDataObject, infer_from_python from .type_codes import * @@ -30,7 +31,7 @@ ] -class ObjectArrayObject: +class ObjectArrayObject(IgniteDataType): """ Array of objects of any type. Its Python representation is tuple(type_id, iterable of any type). @@ -106,11 +107,11 @@ def from_python(cls, value): buffer = bytes(header) for x in value: - buffer += AnyDataObject.from_python(x) + buffer += infer_from_python(x) return buffer -class WrappedDataObject: +class WrappedDataObject(IgniteDataType): """ One or more binary objects can be wrapped in an array. This allows reading, storing, passing and writing objects efficiently without understanding @@ -195,7 +196,7 @@ def build_header(cls): ) -class Map: +class Map(IgniteDataType): """ Dictionary type, payload-only. @@ -273,14 +274,8 @@ def from_python(cls, value, type_id=None): buffer = bytes(header) for k, v in value.items(): - if is_hinted(k): - buffer += k[1].from_python(k[0]) - else: - buffer += AnyDataObject.from_python(k) - if is_hinted(v): - buffer += v[1].from_python(v[0]) - else: - buffer += AnyDataObject.from_python(v) + buffer += infer_from_python(k) + buffer += infer_from_python(v) return buffer @@ -323,7 +318,7 @@ def from_python(cls, value): return super().from_python(value, type_id) -class BinaryObject: +class BinaryObject(IgniteDataType): type_code = TC_COMPLEX_OBJECT USER_TYPE = 0x0001 diff --git a/modules/platforms/python/pyignite/datatypes/internal.py b/modules/platforms/python/pyignite/datatypes/internal.py index a363a5fd766f9..844e0ef9e7fb4 100644 --- a/modules/platforms/python/pyignite/datatypes/internal.py +++ b/modules/platforms/python/pyignite/datatypes/internal.py @@ -389,6 +389,20 @@ def from_python(cls, value): return cls.map_python_type(value).from_python(value) +def infer_from_python(value: Any): + """ + Convert pythonic value to ctypes buffer, type hint-aware. + + :param value: pythonic value or (value, type_hint) tuple, + :return: bytes. + """ + if is_hinted(value): + value, data_type = value + else: + data_type = AnyDataObject + return data_type.from_python(value) + + @attr.s class AnyDataArray(AnyDataObject): """ @@ -454,8 +468,5 @@ def from_python(self, value): buffer = bytes(header) for x in value: - if is_hinted(x): - buffer += x[1].from_python(x[0]) - else: - buffer += super().from_python(x) + buffer += infer_from_python(x) return buffer diff --git a/modules/platforms/python/pyignite/datatypes/null_object.py b/modules/platforms/python/pyignite/datatypes/null_object.py index 9fa1e8feefbdb..a648e30f1bb96 100644 --- a/modules/platforms/python/pyignite/datatypes/null_object.py +++ b/modules/platforms/python/pyignite/datatypes/null_object.py @@ -21,13 +21,14 @@ import ctypes +from .base import IgniteDataType from .type_codes import TC_NULL __all__ = ['Null'] -class Null: +class Null(IgniteDataType): default = None pythonic = type(None) _object_c_type = None diff --git a/modules/platforms/python/pyignite/datatypes/primitive.py b/modules/platforms/python/pyignite/datatypes/primitive.py index 94c8fe3163b12..d1e9f4e4ae566 100644 --- a/modules/platforms/python/pyignite/datatypes/primitive.py +++ b/modules/platforms/python/pyignite/datatypes/primitive.py @@ -16,6 +16,7 @@ import ctypes from pyignite.constants import * +from .base import IgniteDataType __all__ = [ @@ -24,7 +25,7 @@ ] -class Primitive: +class Primitive(IgniteDataType): """ Ignite primitive type. Base type for the following types: diff --git a/modules/platforms/python/pyignite/datatypes/primitive_arrays.py b/modules/platforms/python/pyignite/datatypes/primitive_arrays.py index 83a2b4ceb5b4d..6a931910d4636 100644 --- a/modules/platforms/python/pyignite/datatypes/primitive_arrays.py +++ b/modules/platforms/python/pyignite/datatypes/primitive_arrays.py @@ -16,6 +16,7 @@ import ctypes from pyignite.constants import * +from .base import IgniteDataType from .primitive import * from .type_codes import * @@ -28,7 +29,7 @@ ] -class PrimitiveArray: +class PrimitiveArray(IgniteDataType): """ Base class for array of primitives. Payload-only. """ diff --git a/modules/platforms/python/pyignite/datatypes/primitive_objects.py b/modules/platforms/python/pyignite/datatypes/primitive_objects.py index 4e37ce10c1d70..105acee000b56 100644 --- a/modules/platforms/python/pyignite/datatypes/primitive_objects.py +++ b/modules/platforms/python/pyignite/datatypes/primitive_objects.py @@ -16,6 +16,7 @@ import ctypes from pyignite.constants import * +from .base import IgniteDataType from .type_codes import * @@ -25,7 +26,7 @@ ] -class DataObject: +class DataObject(IgniteDataType): """ Base class for primitive data objects. diff --git a/modules/platforms/python/pyignite/datatypes/standard.py b/modules/platforms/python/pyignite/datatypes/standard.py index 5f3af744c6bd0..cc5b9553bcd08 100644 --- a/modules/platforms/python/pyignite/datatypes/standard.py +++ b/modules/platforms/python/pyignite/datatypes/standard.py @@ -20,6 +20,7 @@ import uuid from pyignite.constants import * +from .base import IgniteDataType from .type_codes import * from .null_object import Null @@ -39,7 +40,7 @@ ] -class StandardObject: +class StandardObject(IgniteDataType): type_code = None @classmethod @@ -58,7 +59,7 @@ def parse(cls, client: 'Client'): return c_type, buffer -class String: +class String(IgniteDataType): """ Pascal-style string: `c_int` counter, followed by count*bytes. UTF-8-encoded, so that one character may take 1 to 4 bytes. @@ -125,7 +126,7 @@ def from_python(cls, value): return bytes(data_object) -class DecimalObject: +class DecimalObject(IgniteDataType): type_code = TC_DECIMAL pythonic = decimal.Decimal default = decimal.Decimal('0.00') @@ -511,7 +512,7 @@ class BinaryEnumObject(EnumObject): type_code = TC_BINARY_ENUM -class StandardArray: +class StandardArray(IgniteDataType): """ Base class for array of primitives. Payload-only. """ diff --git a/modules/platforms/python/pyignite/utils.py b/modules/platforms/python/pyignite/utils.py index a08bc9bae9a6f..1d4298ec8e4f8 100644 --- a/modules/platforms/python/pyignite/utils.py +++ b/modules/platforms/python/pyignite/utils.py @@ -16,6 +16,7 @@ from functools import wraps from typing import Any, Type, Union +from pyignite.datatypes.base import IgniteDataType from .constants import * @@ -47,11 +48,14 @@ def is_hinted(value): return ( isinstance(value, tuple) and len(value) == 2 - and isinstance(value[1], object) + and issubclass(value[1], IgniteDataType) ) def is_wrapped(value: Any) -> bool: + """ + Check if a value is of WrappedDataObject type. + """ return ( type(value) is tuple and len(value) == 2 diff --git a/modules/platforms/python/setup.py b/modules/platforms/python/setup.py index 7419c975e9c1a..583eaa38b31c4 100644 --- a/modules/platforms/python/setup.py +++ b/modules/platforms/python/setup.py @@ -70,7 +70,7 @@ def is_a_requirement(line): setuptools.setup( name='pyignite', - version='0.3.1', + version='0.3.4', python_requires='>={}.{}'.format(*PYTHON_REQUIRED), author='Dmitry Melnichuk', author_email='dmitry.melnichuk@nobitlost.com', diff --git a/modules/platforms/python/tests/test_key_value.py b/modules/platforms/python/tests/test_key_value.py index c569c7724cc72..6b4fb0ea575d2 100644 --- a/modules/platforms/python/tests/test_key_value.py +++ b/modules/platforms/python/tests/test_key_value.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime + from pyignite.api import * -from pyignite.datatypes import IntObject +from pyignite.datatypes import ( + CollectionObject, IntObject, MapObject, TimestampObject, +) def test_put_get(client, cache): @@ -325,3 +329,72 @@ def test_cache_get_size(client, cache): result = cache_get_size(client, cache) assert result.status == 0 assert result.value == 1 + + +def test_put_get_collection(client): + + test_datetime = datetime(year=1996, month=3, day=1) + + cache = client.get_or_create_cache('test_coll_cache') + cache.put( + 'simple', + ( + 1, + [ + (123, IntObject), + 678, + None, + 55.2, + ((test_datetime, 0), TimestampObject), + ] + ), + value_hint=CollectionObject + ) + value = cache.get('simple') + assert value == (1, [123, 678, None, 55.2, (test_datetime, 0)]) + + cache.put( + 'nested', + ( + 1, + [ + 123, + ((1, [456, 'inner_test_string', 789]), CollectionObject), + 'outer_test_string', + ] + ), + value_hint=CollectionObject + ) + value = cache.get('nested') + assert value == ( + 1, + [ + 123, + (1, [456, 'inner_test_string', 789]), + 'outer_test_string' + ] + ) + + +def test_put_get_map(client): + + cache = client.get_or_create_cache('test_map_cache') + + cache.put( + 'test_map', + ( + MapObject.HASH_MAP, + { + (123, IntObject): 'test_data', + 456: ((1, [456, 'inner_test_string', 789]), CollectionObject), + 'test_key': 32.4, + } + ), + value_hint=MapObject + ) + value = cache.get('test_map') + assert value == (MapObject.HASH_MAP, { + 123: 'test_data', + 456: (1, [456, 'inner_test_string', 789]), + 'test_key': 32.4, + }) From 8347aef54e87b669c37557b88857d04f5c2fd415 Mon Sep 17 00:00:00 2001 From: Max-Pudov Date: Mon, 26 Nov 2018 17:46:56 +0300 Subject: [PATCH 174/403] IGNITE-9818 Fix javadoc for annotation AffinityKeyMapped - Fixes #4927. Signed-off-by: Dmitriy Pavlov (cherry picked from commit 6fd6c320ed0f79dc71a54981982361ab8dc30f44) --- .../org/apache/ignite/cache/affinity/AffinityKeyMapped.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKeyMapped.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKeyMapped.java index 8b19338571805..e7e9eba1ccfd7 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKeyMapped.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/AffinityKeyMapped.java @@ -91,8 +91,8 @@ * is otherwise known as {@code Collocation Of Computations And Data}. In this case, * {@code @AffinityKeyMapped} annotation allows to specify a routing affinity key for a * {@link org.apache.ignite.compute.ComputeJob} or any other grid computation, such as {@link Runnable}, - * {@link Callable}, or {@link org.apache.ignite.lang.IgniteClosure}. It should be attached to a method or - * field that provides affinity key for the computation. Only one annotation per class is allowed. + * {@link Callable}, or {@link org.apache.ignite.lang.IgniteClosure}. It should be attached to a field + * that provides affinity key for the computation. Only one annotation per class is allowed. * Whenever such annotation is detected, then {@link org.apache.ignite.spi.loadbalancing.LoadBalancingSpi} * will be bypassed, and computation will be routed to the grid node where the specified affinity key is cached. *

From b3cd4d1df1f4ba991c4dd58645d718186c121705 Mon Sep 17 00:00:00 2001 From: Roman Guseinov Date: Mon, 26 Nov 2018 16:57:47 +0300 Subject: [PATCH 175/403] IGNITE-9937 Primary response error can be lost due to unwrapping a key - Fixes #5078. Signed-off-by: Pavel Kovalenko --- .../dht/atomic/GridDhtAtomicCache.java | 30 ++++- .../GridNearAtomicAbstractUpdateFuture.java | 43 +++++- .../cache/store/CacheStoreWriteErrorTest.java | 127 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite.java | 2 + 4 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreWriteErrorTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index c7b6677ebdbdc..09dc4dfe119f1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -34,6 +34,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.binary.BinaryInvalidTypeException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.NodeStoppingException; @@ -2725,6 +2726,8 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) { final GridDhtAtomicAbstractUpdateFuture dhtFut = dhtUpdRes.dhtFuture(); + Collection failedToUnwrapKeys = null; + // Avoid iterator creation. for (int i = 0; i < entries.size(); i++) { GridDhtCacheEntry entry = entries.get(i); @@ -2737,9 +2740,26 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) { continue; } - if (storeErr != null && - storeErr.failedKeys().contains(entry.key().value(ctx.cacheObjectContext(), false))) - continue; + if (storeErr != null) { + Object key = entry.key(); + + try { + key = entry.key().value(ctx.cacheObjectContext(), false); + } + catch (BinaryInvalidTypeException e) { + if (log.isDebugEnabled()) { + if (failedToUnwrapKeys == null) + failedToUnwrapKeys = new ArrayList<>(); + + // To limit keys count in log message. + if (failedToUnwrapKeys.size() < 5) + failedToUnwrapKeys.add(key); + } + } + + if (storeErr.failedKeys().contains(key)) + continue; + } try { // We are holding java-level locks on entries at this point. @@ -2868,6 +2888,10 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) { dhtUpdRes.processedEntriesCount(firstEntryIdx + i + 1); } + if (failedToUnwrapKeys != null) { + log.warning("Failed to get values of keys: " + failedToUnwrapKeys + + " (the binary objects will be used instead)."); + } } catch (IgniteCheckedException e) { res.addFailedKeys(putMap != null ? putMap.keySet() : rmvKeys, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java index 983b18ac38ba0..f4c9b5cf22645 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java @@ -30,6 +30,7 @@ import javax.cache.expiry.ExpiryPolicy; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.binary.BinaryInvalidTypeException; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; @@ -400,10 +401,46 @@ final void onPrimaryError(GridNearAtomicAbstractUpdateRequest req, GridNearAtomi Collection keys = new ArrayList<>(keys0.size()); - for (KeyCacheObject key : keys0) - keys.add(cctx.cacheObjectContext().unwrapBinaryIfNeeded(key, keepBinary, false)); + Collection failedToUnwrapKeys = null; - err.add(keys, res.error(), req.topologyVersion()); + Exception suppressedErr = null; + + for (KeyCacheObject key : keys0) { + try { + keys.add(cctx.cacheObjectContext().unwrapBinaryIfNeeded(key, keepBinary, false)); + } + catch (BinaryInvalidTypeException e) { + keys.add(cctx.toCacheKeyObject(key)); + + if (log.isDebugEnabled()) { + if (failedToUnwrapKeys == null) + failedToUnwrapKeys = new ArrayList<>(); + + // To limit keys count in log message. + if (failedToUnwrapKeys.size() < 5) + failedToUnwrapKeys.add(key); + } + + suppressedErr = e; + } + catch (Exception e) { + keys.add(cctx.toCacheKeyObject(key)); + + suppressedErr = e; + } + } + + if (failedToUnwrapKeys != null) { + log.warning("Failed to unwrap keys: " + failedToUnwrapKeys + + " (the binary objects will be used instead)."); + } + + IgniteCheckedException error = res.error(); + + if (suppressedErr != null) + error.addSuppressed(suppressedErr); + + err.add(keys, error, req.topologyVersion()); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreWriteErrorTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreWriteErrorTest.java new file mode 100644 index 0000000000000..fce1f5d65c3d0 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreWriteErrorTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cache.store; + +import com.google.common.base.Throwables; +import java.util.HashMap; +import java.util.concurrent.Callable; +import javax.cache.Cache; +import javax.cache.configuration.FactoryBuilder; +import javax.cache.integration.CacheLoaderException; +import javax.cache.integration.CacheWriterException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * This class tests handling exceptions from {@link CacheStore#write(Cache.Entry)}. + */ +public class CacheStoreWriteErrorTest extends GridCommonAbstractTest { + /** */ + public static final String CACHE_NAME = "cache"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + CacheConfiguration cacheCfg = new CacheConfiguration(CACHE_NAME) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC) + .setCacheStoreFactory(FactoryBuilder.factoryOf(ThrowableCacheStore.class)) + .setWriteThrough(true) + .setStoreKeepBinary(true); + + return super.getConfiguration(gridName) + .setCacheConfiguration(cacheCfg); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * Checks primary error while saving batch with one entry. + */ + public void testPrimaryErrorForBatchSize1() { + checkPrimaryError(1); + } + + /** + * Checks primary error while saving batch with two entries. + */ + public void testPrimaryErrorForBatchSize2() { + checkPrimaryError(2); + } + + /** + * Checks that primary error ({@link CacheWriterException}) is not lost due to unwrapping a key. + * + * @param batchSize Batch size. + */ + private void checkPrimaryError(int batchSize) { + Throwable t = GridTestUtils.assertThrows(log, + new Callable() { + @Override public Object call() throws Exception { + try (Ignite grid = startGrid()) { + IgniteCache cache = grid.cache(CACHE_NAME); + + HashMap batch = new HashMap<>(); + + for (int i = 0; i < batchSize; i++) { + BinaryObject key = grid.binary().builder("KEY_TYPE_NAME").setField("id", i).build(); + + batch.put(key, "VALUE"); + } + + cache.putAllAsync(batch).get(); + } + + return null; + } + }, CacheWriterException.class, null); + + assertTrue("Stacktrace should contain the message of the original exception", + Throwables.getStackTraceAsString(t).contains(ThrowableCacheStore.EXCEPTION_MESSAGE)); + } + + /** + * {@link CacheStore} implementation which throws {@link RuntimeException} for every write operation. + */ + public static class ThrowableCacheStore extends CacheStoreAdapter { + /** */ + private static final String EXCEPTION_MESSAGE = "WRITE CACHE STORE EXCEPTION"; + + /** {@inheritDoc} */ + @Override public Object load(Object o) throws CacheLoaderException { + return null; + } + + /** {@inheritDoc} */ + @Override public void write(Cache.Entry entry) throws CacheWriterException { + throw new RuntimeException(EXCEPTION_MESSAGE); + } + + /** {@inheritDoc} */ + @Override public void delete(Object o) throws CacheWriterException { + // No-op. + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java index 394624d5c5724..905428482b4f4 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java @@ -22,6 +22,7 @@ import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest; import org.apache.ignite.cache.IgniteWarmupClosureSelfTest; import org.apache.ignite.cache.store.CacheStoreReadFromBackupTest; +import org.apache.ignite.cache.store.CacheStoreWriteErrorTest; import org.apache.ignite.cache.store.CacheTransactionalStoreReadFromBackupTest; import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest; import org.apache.ignite.cache.store.GridCacheLoadOnlyStoreAdapterSelfTest; @@ -358,6 +359,7 @@ public static TestSuite suite(Set ignoredTests) throws Exception { suite.addTestSuite(GridStoreLoadCacheTest.class); suite.addTestSuite(CacheStoreReadFromBackupTest.class); + suite.addTestSuite(CacheStoreWriteErrorTest.class); suite.addTestSuite(CacheTransactionalStoreReadFromBackupTest.class); //suite.addTestSuite(CacheAtomicSingleMessageCountSelfTest.class); From 34e7fa98b824de3bff619cdad5498aade1205ec5 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Tue, 27 Nov 2018 11:45:12 +0300 Subject: [PATCH 176/403] IGNITE-10002: MVCC: Create "Cache 2" test suite for MVCC mode. This closes #5198. (cherry picked from commit 7577c8770eaf29e21f1d46f650af1d8fbd891c42) --- .../apache/ignite/IgniteSystemProperties.java | 3 + .../cache/mvcc/MvccProcessorImpl.java | 11 + .../cache/CacheConcurrentReadThroughTest.java | 8 + .../CacheEnumOperationsAbstractTest.java | 16 +- .../CrossCacheTxRandomOperationsTest.java | 23 +- .../cache/GridCacheBasicApiAbstractTest.java | 27 ++ .../GridCacheBasicStoreAbstractTest.java | 8 + .../GridCacheEvictionEventAbstractTest.java | 9 + .../GridCacheFinishPartitionsSelfTest.java | 8 + ...CacheGetAndTransformStoreAbstractTest.java | 8 + .../cache/GridCacheOffheapUpdateSelfTest.java | 17 +- .../GridCachePartitionedGetSelfTest.java | 6 + .../GridCacheVariableTopologySelfTest.java | 9 + ...IgniteCacheEntryProcessorNodeJoinTest.java | 110 +++--- .../cache/IgniteCacheIncrementTxTest.java | 7 + .../cache/IgniteCacheNoSyncForGetTest.java | 8 + .../IgniteClientCacheStartFailoverTest.java | 37 ++ ...IgniteMvccTxMultiThreadedAbstractTest.java | 121 +++++++ ...gniteMvccTxSingleThreadedAbstractTest.java | 50 +++ .../cache/IgniteNearClientCacheCloseTest.java | 22 ++ .../cache/IgniteTxAbstractTest.java | 16 +- .../IgniteTxMultiThreadedAbstractTest.java | 1 - ...cheLoadingConcurrentGridStartSelfTest.java | 8 + .../CacheLockReleaseNodeLeaveTest.java | 8 + .../GridCacheAbstractPrimarySyncSelfTest.java | 4 + .../GridCacheBasicOpAbstractTest.java | 8 + .../GridCacheLockAbstractTest.java | 8 + .../GridCacheMultiNodeAbstractTest.java | 5 + .../GridCacheMultiNodeLockAbstractTest.java | 9 + .../GridCacheNodeFailureAbstractTest.java | 8 + ...dCachePartitionNotLoadedEventSelfTest.java | 12 +- ...arDisabledMvccTxMultiThreadedSelfTest.java | 31 ++ ...idCachePreloadRestartAbstractSelfTest.java | 12 + .../GridCacheTransformEventSelfTest.java | 64 +++- ...teCacheClientNodeChangingTopologyTest.java | 8 + .../IgniteCacheTxIteratorSelfTest.java | 26 +- .../IgniteMvccTxTimeoutAbstractTest.java | 146 ++++++++ .../IgniteTxTimeoutAbstractTest.java | 39 ++- ...ColocatedMvccTxSingleThreadedSelfTest.java | 85 +++++ .../dht/GridCacheDhtEntrySelfTest.java | 5 + .../dht/GridCacheDhtMappingSelfTest.java | 8 + .../GridCacheDhtPreloadDisabledSelfTest.java | 8 + ...dCacheDhtPreloadMultiThreadedSelfTest.java | 3 + .../GridCacheDhtPreloadPutGetSelfTest.java | 7 + .../dht/GridCacheDhtPreloadSelfTest.java | 2 - .../IgniteCacheClearDuringRebalanceTest.java | 9 + ...titionedBackupNodeFailureRecoveryTest.java | 2 +- .../near/GridCacheNearClientHitTest.java | 5 +- .../near/GridCacheNearMultiGetSelfTest.java | 73 ++-- .../near/GridCacheNearMultiNodeSelfTest.java | 10 + .../near/GridCacheNearOneNodeSelfTest.java | 11 +- ...GridCacheNearPartitionedClearSelfTest.java | 8 + .../GridCacheNearReaderPreloadSelfTest.java | 8 + .../near/GridCacheNearReadersSelfTest.java | 10 +- ...artitionedBasicStoreMultiNodeSelfTest.java | 6 + .../GridCachePartitionedEventSelfTest.java | 8 + ...tionedExplicitLockNodeFailureSelfTest.java | 8 + ...GridCachePartitionedLoadCacheSelfTest.java | 8 + ...artitionedMvccTxMultiThreadedSelfTest.java | 101 ++++++ ...rtitionedMvccTxSingleThreadedSelfTest.java | 84 +++++ ...CachePartitionedMvccTxTimeoutSelfTest.java | 47 +++ ...hePartitionedPreloadLifecycleSelfTest.java | 1 - ...GridCachePartitionedTxTimeoutSelfTest.java | 21 -- .../near/GridNearCacheStoreUpdateTest.java | 9 + .../near/NearCacheSyncUpdateTest.java | 17 + .../near/NoneRebalanceModeSelfTest.java | 9 + .../GridCacheReplicatedTxTimeoutSelfTest.java | 18 - .../local/GridCacheLocalBasicApiSelfTest.java | 8 + .../GridCacheLocalBasicStoreSelfTest.java | 8 + .../local/GridCacheLocalEventSelfTest.java | 9 + .../GridCacheLocalEvictionEventSelfTest.java | 8 + ...acheLocalGetAndTransformStoreSelfTest.java | 8 + .../GridCacheLocalIsolatedNodesSelfTest.java | 8 + .../local/GridCacheLocalLoadAllSelfTest.java | 8 + .../local/GridCacheLocalLockSelfTest.java | 8 + .../GridCacheLocalMultithreadedSelfTest.java | 8 + ...GridCacheLocalTxMultiThreadedSelfTest.java | 8 + ...ridCacheLocalTxSingleThreadedSelfTest.java | 8 + .../GridCacheLocalTxTimeoutSelfTest.java | 8 + .../testframework/MvccFeatureChecker.java | 137 ++++++++ .../testsuites/IgniteCacheMvccTestSuite2.java | 197 +++++++++++ .../testsuites/IgniteCacheTestSuite2.java | 321 ++++++++++-------- .../testsuites/IgniteCacheTestSuite6.java | 3 - 83 files changed, 1944 insertions(+), 323 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxMultiThreadedAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxSingleThreadedAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteMvccTxTimeoutAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheColocatedMvccTxSingleThreadedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxMultiThreadedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxSingleThreadedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxTimeoutSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 943583bcf1d72..5d441e601067a 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1031,6 +1031,9 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_ALLOW_START_CACHES_IN_PARALLEL = "IGNITE_ALLOW_START_CACHES_IN_PARALLEL"; + /** For test purposes only. Force Mvcc mode. */ + public static final String IGNITE_FORCE_MVCC_MODE_IN_TESTS = "IGNITE_FORCE_MVCC_MODE_IN_TESTS"; + /** * Allows to log additional information about all restored partitions after binary and logical recovery phases. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java index e8f3c5386624e..b3e3e108beeef 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java @@ -34,6 +34,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -106,6 +107,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_JOINED; @@ -133,6 +135,10 @@ */ @SuppressWarnings("serial") public class MvccProcessorImpl extends GridProcessorAdapter implements MvccProcessor, DatabaseLifecycleListener { + /** */ + private static final boolean FORCE_MVCC = + IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, false); + /** */ private static final IgniteProductVersion MVCC_SUPPORTED_SINCE = IgniteProductVersion.fromString("2.7.0"); @@ -262,6 +268,11 @@ public MvccProcessorImpl(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void preProcessCacheConfiguration(CacheConfiguration ccfg) { + if (FORCE_MVCC && ccfg.getAtomicityMode() == TRANSACTIONAL && !CU.isSystemCache(ccfg.getName())) { + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + ccfg.setNearConfiguration(null); + } + if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) { if (!mvccSupported) throw new IgniteException("Cannot start MVCC transactional cache. " + diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java index 130280dd3fcb0..b2842e09730ae 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -50,6 +51,13 @@ public class CacheConcurrentReadThroughTest extends GridCommonAbstractTest { /** */ private boolean client; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEnumOperationsAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEnumOperationsAbstractTest.java index 148b60e473060..0cffeb610c0dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEnumOperationsAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEnumOperationsAbstractTest.java @@ -35,6 +35,8 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -92,7 +94,19 @@ public void testAtomic() throws Exception { * @throws Exception If failed. */ public void testTx() throws Exception { - CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, ATOMIC); + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL); + + enumOperations(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTx() throws Exception { + if (!singleNode()) + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, 1, TRANSACTIONAL_SNAPSHOT); enumOperations(ccfg); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java index cc9823bc63e7e..cba3771665ff5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -92,6 +93,9 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + if (nearCacheEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + super.beforeTestsStarted(); startGridsMultiThreaded(GRID_CNT - 1); @@ -194,6 +198,13 @@ protected void createCache(CacheMode cacheMode, private void txOperations(CacheMode cacheMode, CacheWriteSynchronizationMode writeSync, boolean crossCacheTx) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + assert !nearCacheEnabled(); + + if(writeSync != CacheWriteSynchronizationMode.FULL_SYNC) + return; + } + Ignite ignite = ignite(0); try { @@ -203,12 +214,14 @@ private void txOperations(CacheMode cacheMode, txOperations(PESSIMISTIC, REPEATABLE_READ, crossCacheTx, false); txOperations(PESSIMISTIC, REPEATABLE_READ, crossCacheTx, true); - txOperations(OPTIMISTIC, REPEATABLE_READ, crossCacheTx, false); - txOperations(OPTIMISTIC, REPEATABLE_READ, crossCacheTx, true); + if(!MvccFeatureChecker.forcedMvcc()) { + txOperations(OPTIMISTIC, REPEATABLE_READ, crossCacheTx, false); + txOperations(OPTIMISTIC, REPEATABLE_READ, crossCacheTx, true); - if (writeSync == FULL_SYNC) { - txOperations(OPTIMISTIC, SERIALIZABLE, crossCacheTx, false); - txOperations(OPTIMISTIC, SERIALIZABLE, crossCacheTx, true); + if (writeSync == FULL_SYNC) { + txOperations(OPTIMISTIC, SERIALIZABLE, crossCacheTx, false); + txOperations(OPTIMISTIC, SERIALIZABLE, crossCacheTx, true); + } } } finally { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicApiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicApiAbstractTest.java index f766d01517d65..0b7de6e501704 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicApiAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicApiAbstractTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestThread; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -89,6 +90,8 @@ protected GridCacheBasicApiAbstractTest() { * @throws Exception If test failed. */ public void testBasicLock() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); Lock lock = cache.lock(1); @@ -106,6 +109,8 @@ public void testBasicLock() throws Exception { * @throws IgniteCheckedException If test failed. */ public void testSingleLockReentry() throws IgniteCheckedException { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); Lock lock = cache.lock(1); @@ -134,6 +139,8 @@ public void testSingleLockReentry() throws IgniteCheckedException { * @throws Exception If test failed. */ public void testReentry() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); Lock lock = cache.lock(1); @@ -173,6 +180,8 @@ public void testReentry() throws Exception { * */ public void testInterruptLock() throws InterruptedException { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + final IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); final Lock lock = cache.lock(1); @@ -217,6 +226,8 @@ public void testInterruptLock() throws InterruptedException { * */ public void testInterruptLockWithTimeout() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + final IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); startGrid(1); @@ -276,6 +287,8 @@ public void testInterruptLockWithTimeout() throws Exception { * @throws IgniteCheckedException If test failed. */ public void testManyLockReentries() throws IgniteCheckedException { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); Integer key = 1; @@ -319,6 +332,8 @@ public void testManyLockReentries() throws IgniteCheckedException { * @throws Exception If test failed. */ public void testLockMultithreaded() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + final IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); final CountDownLatch l1 = new CountDownLatch(1); @@ -437,6 +452,9 @@ public void testLockMultithreaded() throws Exception { * @throws Exception If error occur. */ public void testBasicOps() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); CountDownLatch latch = new CountDownLatch(1); @@ -498,6 +516,9 @@ public void testBasicOps() throws Exception { * @throws Exception If error occur. */ public void testBasicOpsWithReentry() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); int key = (int)System.currentTimeMillis(); @@ -570,6 +591,8 @@ public void testBasicOpsWithReentry() throws Exception { * @throws Exception If test failed. */ public void testMultiLocks() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); Collection keys = Arrays.asList(1, 2, 3); @@ -625,6 +648,10 @@ public void testGetPutRemove() throws IgniteCheckedException { * @throws Exception In case of error. */ public void testPutWithExpiration() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); CacheEventListener lsnr = new CacheEventListener(new CountDownLatch(1)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java index 84ce5385256c3..9de410a225169 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheBasicStoreAbstractTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.Nullable; @@ -53,6 +54,13 @@ public abstract class GridCacheBasicStoreAbstractTest extends GridCommonAbstract /** Cache store. */ private static final GridCacheTestStore store = new GridCacheTestStore(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionEventAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionEventAbstractTest.java index 554a7a9ed3db4..b2c87a9912606 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionEventAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionEventAbstractTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.events.EventType.EVT_CACHE_ENTRY_EVICTED; @@ -49,6 +50,14 @@ public abstract class GridCacheEvictionEventAbstractTest extends GridCommonAbstr /** */ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFinishPartitionsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFinishPartitionsSelfTest.java index 9732272d5f1e1..d60e4c665a3a2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFinishPartitionsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFinishPartitionsSelfTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -56,6 +57,13 @@ public class GridCacheFinishPartitionsSelfTest extends GridCacheAbstractSelfTest return GRID_CNT; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { grid = (IgniteKernal)grid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheGetAndTransformStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheGetAndTransformStoreAbstractTest.java index f140945656e26..af9751f7b9a52 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheGetAndTransformStoreAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheGetAndTransformStoreAbstractTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -48,6 +49,13 @@ public abstract class GridCacheGetAndTransformStoreAbstractTest extends GridComm /** Cache store. */ private static final GridCacheTestStore store = new GridCacheTestStore(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffheapUpdateSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffheapUpdateSelfTest.java index b8f68585a2a03..cf1641764adb1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffheapUpdateSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffheapUpdateSelfTest.java @@ -127,12 +127,19 @@ public void testReadEvictedPartition() throws Exception { assertEquals(10, cache.get(key)); - try (Transaction ignored = grid.transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - assertEquals(10, cache.get(key)); + if(((IgniteCacheProxy)cache).context().config().getAtomicityMode() != CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) { + try (Transaction ignored = grid.transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + assertEquals(10, cache.get(key)); + } + + try (Transaction ignored = grid.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + assertEquals(10, cache.get(key)); + } } - - try (Transaction ignored = grid.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { - assertEquals(10, cache.get(key)); + else { + try (Transaction ignored = grid.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + assertEquals(10, cache.get(key)); + } } } finally { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java index f5cc65b9b8ec5..d2581e0614c9c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedGetSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -129,6 +130,11 @@ public void testGetFromPrimaryNode() throws Exception { * @throws Exception If failed. */ public void testGetFromBackupNode() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10274"); + + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + for (int i = 0; i < GRID_CNT; i++) { IgniteCache c = grid(i).cache(DEFAULT_CACHE_NAME); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVariableTopologySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVariableTopologySelfTest.java index 25817a191c3ee..b9f7a3577fdb6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVariableTopologySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVariableTopologySelfTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionRollbackException; @@ -52,6 +53,14 @@ public class GridCacheVariableTopologySelfTest extends GridCommonAbstractTest { /** */ private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7388"); + + super.setUp(); + } + /** Constructs test. */ public GridCacheVariableTopologySelfTest() { super(/* don't start grid */ false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java index ec982949efb3c..89d771ee43dd6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java @@ -45,6 +45,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; @@ -111,7 +112,7 @@ protected CacheAtomicityMode atomicityMode() { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - startGrids(GRID_CNT); + startGridsMultiThreaded(GRID_CNT, true); } /** {@inheritDoc} */ @@ -137,10 +138,13 @@ public void testAllEntryProcessorNodeJoin() throws Exception { * @throws Exception If failed. */ public void testEntryProcessorNodeLeave() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10254"); + startGrid(GRID_CNT); // TODO: IGNITE-1525 (test fails with one-phase commit). - boolean createCache = atomicityMode() == TRANSACTIONAL; + boolean createCache = atomicityMode() != ATOMIC; String cacheName = DEFAULT_CACHE_NAME; @@ -208,32 +212,34 @@ public void testEntryProcessorNodeLeave() throws Exception { * @throws Exception If failed. */ private void checkEntryProcessorNodeJoin(boolean invokeAll) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + final AtomicBoolean stop = new AtomicBoolean(); final AtomicReference error = new AtomicReference<>(); final int started = 6; - try { - IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Runnable() { - @Override public void run() { - try { - for (int i = 0; i < started; i++) { - U.sleep(1_000); + IgniteInternalFuture fut = GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + try { + for (int i = 0; i < started && !stop.get(); i++) { + U.sleep(1_000); + if (!stop.get()) startGrid(GRID_CNT + i); - } - } - catch (Exception e) { - error.compareAndSet(null, e); } } - }, 1, "starter"); + catch (Exception e) { + error.compareAndSet(null, e); + } + } + }, "starter"); + try { try { checkIncrement(DEFAULT_CACHE_NAME, invokeAll, null, null); } finally { - stop.set(true); - fut.get(getTestTimeout()); } @@ -247,8 +253,10 @@ private void checkEntryProcessorNodeJoin(boolean invokeAll) throws Exception { } } finally { - for (int i = 0; i < started; i++) - stopGrid(GRID_CNT + i); + stop.set(true); + + if (!fut.isDone()) + fut.cancel(); } } @@ -327,51 +335,51 @@ public void testReplaceNodeJoin() throws Exception { final AtomicReference error = new AtomicReference<>(); final int started = 6; - try { - int keys = 100; + int keys = 100; - final AtomicBoolean done = new AtomicBoolean(false); + final AtomicBoolean stop = new AtomicBoolean(false); - for (int i = 0; i < keys; i++) - ignite(0).cache(DEFAULT_CACHE_NAME).put(i, 0); + for (int i = 0; i < keys; i++) + ignite(0).cache(DEFAULT_CACHE_NAME).put(i, 0); - IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Runnable() { - @Override public void run() { - try { - for (int i = 0; i < started; i++) { - U.sleep(1_000); + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Runnable() { + @Override public void run() { + try { + for (int i = 0; i < started && !stop.get(); i++) { + U.sleep(1_000); - IgniteEx grid = startGrid(GRID_CNT + i); + if (stop.get()) + continue; - info("Test started grid [idx=" + (GRID_CNT + i) + ", nodeId=" + grid.localNode().id() + ']'); - } - } - catch (Exception e) { - error.compareAndSet(null, e); - } - finally { - done.set(true); + IgniteEx grid = startGrid(GRID_CNT + i); + + info("Test started grid [idx=" + (GRID_CNT + i) + ", nodeId=" + grid.localNode().id() + ']'); } } - }, 1, "starter"); + catch (Exception e) { + error.compareAndSet(null, e); + } + finally { + stop.set(true); + } + } + }, 1, "starter"); + try { int updVal = 0; - try { - while (!done.get()) { - info("Will put: " + (updVal + 1)); + while (!stop.get()) { + info("Will put: " + (updVal + 1)); - for (int i = 0; i < keys; i++) - assertTrue("Failed [key=" + i + ", oldVal=" + updVal+ ']', - ignite(0).cache(DEFAULT_CACHE_NAME).replace(i, updVal, updVal + 1)); + for (int i = 0; i < keys; i++) + assertTrue("Failed [key=" + i + ", oldVal=" + updVal+ ']', + ignite(0).cache(DEFAULT_CACHE_NAME).replace(i, updVal, updVal + 1)); - updVal++; - } - } - finally { - fut.get(getTestTimeout()); + updVal++; } + fut.get(getTestTimeout()); + for (int i = 0; i < keys; i++) { for (int g = 0; g < GRID_CNT + started; g++) { Integer val = ignite(g).cache(DEFAULT_CACHE_NAME).get(i); @@ -386,8 +394,10 @@ public void testReplaceNodeJoin() throws Exception { } } finally { - for (int i = 0; i < started; i++) - stopGrid(GRID_CNT + i); + stop.set(true); + + if (!fut.isDone()) + fut.cancel(); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java index b3a505597b8dd..918d655e013ac 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java @@ -36,6 +36,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -88,6 +89,9 @@ public void testIncrementTxTopologyChange0() throws Exception { * @throws Exception If failed. */ public void testIncrementTxTopologyChange1() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10264"); + nodeJoin(cacheConfiguration(1)); } @@ -95,6 +99,9 @@ public void testIncrementTxTopologyChange1() throws Exception { * @throws Exception If failed. */ public void testIncrementTxTopologyChange2() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10264"); + nodeJoin(cacheConfiguration(2)); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java index b14fecc5d4a27..6324df4930d46 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNoSyncForGetTest.java @@ -47,6 +47,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** @@ -102,6 +103,13 @@ public void testTxGet() throws Exception { getTest(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testMvccTxGet() throws Exception { + getTest(TRANSACTIONAL_SNAPSHOT); + } + /** * @param atomicityMode Cache atomicity mode. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java index a2d9da719ce98..fa7d0a879ec73 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java @@ -56,6 +56,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** @@ -81,6 +82,13 @@ public class IgniteClientCacheStartFailoverTest extends GridCommonAbstractTest { return cfg; } + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + client = false; + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); @@ -102,6 +110,13 @@ public void testClientStartCoordinatorFailsTx() throws Exception { clientStartCoordinatorFails(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testClientStartCoordinatorFailsMvccTx() throws Exception { + clientStartCoordinatorFails(TRANSACTIONAL_SNAPSHOT); + } + /** * @param atomicityMode Cache atomicity mode. * @throws Exception If failed. @@ -163,6 +178,16 @@ public void testClientStartLastServerFailsTx() throws Exception { clientStartLastServerFails(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testClientStartLastServerFailsMvccTx() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10262"); + + clientStartLastServerFails(TRANSACTIONAL_SNAPSHOT); + } + + /** * @param atomicityMode Cache atomicity mode. * @throws Exception If failed. @@ -547,6 +572,18 @@ private List startCaches(Ignite node, int keys) { cache.putAll(map); } + //TODO: uncomment TRANSACTIONAL_SNAPSHOT cache creation when IGNITE-9470 will be fixed. + /* for (int i = 0; i < 3; i++) { + CacheConfiguration ccfg = cacheConfiguration("mvcc-" + i, TRANSACTIONAL_SNAPSHOT, i); + + IgniteCache cache = node.createCache(ccfg); + + cacheNames.add(ccfg.getName()); + + cache.putAll(map); + }*/ + + return cacheNames; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxMultiThreadedAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxMultiThreadedAbstractTest.java new file mode 100644 index 0000000000000..9e4609b081084 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxMultiThreadedAbstractTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests for local transactions. + */ +public abstract class IgniteMvccTxMultiThreadedAbstractTest extends IgniteTxAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); + + super.beforeTestsStarted(); + } + + /** + * @return Thread count. + */ + protected abstract int threadCount(); + + /** + /** + * @throws IgniteCheckedException If test failed. + */ + public void testPessimisticRepeatableReadCommitMultithreaded() throws Exception { + checkCommitMultithreaded(PESSIMISTIC, REPEATABLE_READ); + + finalChecks(); + } + + /** + * @throws IgniteCheckedException If test failed. + */ + public void testPessimisticRepeatableReadRollbackMultithreaded() throws Exception { + checkRollbackMultithreaded(PESSIMISTIC, REPEATABLE_READ); + + finalChecks(); + } + + /** + * @param concurrency Concurrency. + * @param isolation Isolation. + * @throws Exception If check failed. + */ + protected void checkCommitMultithreaded(final TransactionConcurrency concurrency, + final TransactionIsolation isolation) throws Exception { + GridTestUtils.runMultiThreaded(new Callable() { + @Nullable @Override public Object call() throws Exception { + Thread t = Thread.currentThread(); + + t.setName(t.getName() + "-id-" + t.getId()); + + info("Starting commit thread: " + Thread.currentThread().getName()); + + try { + checkCommit(concurrency, isolation); + } + finally { + info("Finished commit thread: " + Thread.currentThread().getName()); + } + + return null; + } + }, threadCount(), concurrency + "-" + isolation); + } + + /** + * @param concurrency Concurrency. + * @param isolation Isolation. + * @throws Exception If check failed. + */ + protected void checkRollbackMultithreaded(final TransactionConcurrency concurrency, + final TransactionIsolation isolation) throws Exception { + final ConcurrentMap map = new ConcurrentHashMap<>(); + GridTestUtils.runMultiThreaded(new Callable() { + @Nullable @Override public Object call() throws Exception { + Thread t = Thread.currentThread(); + + t.setName(t.getName() + "-id-" + t.getId()); + + info("Starting rollback thread: " + Thread.currentThread().getName()); + + try { + checkRollback(map, concurrency, isolation); + + return null; + } + finally { + info("Finished rollback thread: " + Thread.currentThread().getName()); + } + } + }, threadCount(), concurrency + "-" + isolation); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxSingleThreadedAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxSingleThreadedAbstractTest.java new file mode 100644 index 0000000000000..b9e7cee2547e0 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMvccTxSingleThreadedAbstractTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.IgniteCheckedException; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests for local transactions. + */ +public abstract class IgniteMvccTxSingleThreadedAbstractTest extends IgniteTxAbstractTest { + /** + * @throws IgniteCheckedException If test failed. + */ + public void testPessimisticRepeatableReadCommit() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10261"); + + checkCommit(PESSIMISTIC, REPEATABLE_READ); + + finalChecks(); + } + + /** + * @throws IgniteCheckedException If test failed. + */ + public void testPessimisticRepeatableReadRollback() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10261"); + + checkRollback(PESSIMISTIC, REPEATABLE_READ); + + finalChecks(); + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteNearClientCacheCloseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteNearClientCacheCloseTest.java index e7ab805c3de7c..0ca0f3ed8db48 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteNearClientCacheCloseTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteNearClientCacheCloseTest.java @@ -37,10 +37,12 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** @@ -107,6 +109,26 @@ public void testNearCacheCloseTx2() throws Exception { nearCacheClose(4, true, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testNearCacheCloseMvccTx1() throws Exception { + nearCacheClose(1, false, TRANSACTIONAL_SNAPSHOT); + + if (MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) + nearCacheClose(1, true, TRANSACTIONAL_SNAPSHOT); + } + + /** + * @throws Exception If failed. + */ + public void testNearCacheCloseMvccTx2() throws Exception { + nearCacheClose(4, false, TRANSACTIONAL_SNAPSHOT); + + if (MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) + nearCacheClose(4, true, TRANSACTIONAL_SNAPSHOT); + } + /** * @param srvs Number of server nodes. * @param srvNearCache {@code True} to enable near cache on server nodes. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxAbstractTest.java index 1830db03c367d..3ae4224f722f8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxAbstractTest.java @@ -57,7 +57,7 @@ abstract class IgniteTxAbstractTest extends GridCommonAbstractTest { private static final AtomicInteger cntr = new AtomicInteger(); /** */ - private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + private final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); /** * Start grid by default. @@ -115,12 +115,16 @@ private void debug(String msg) { info(msg); } - /** - * @throws Exception If failed. - */ + /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { - for (int i = 0; i < gridCount(); i++) - startGrid(i); + startGridsMultiThreaded(gridCount()); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxMultiThreadedAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxMultiThreadedAbstractTest.java index 5a1a1db2fdafe..fec3d07f8e05d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxMultiThreadedAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxMultiThreadedAbstractTest.java @@ -86,7 +86,6 @@ protected void checkCommitMultithreaded(final TransactionConcurrency concurrency protected void checkRollbackMultithreaded(final TransactionConcurrency concurrency, final TransactionIsolation isolation) throws Exception { final ConcurrentMap map = new ConcurrentHashMap<>(); - GridTestUtils.runMultiThreaded(new Callable() { @Nullable @Override public Object call() throws Exception { Thread t = Thread.currentThread(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java index cbd012443de9b..115044e463fad 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java @@ -48,6 +48,7 @@ import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -78,6 +79,13 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT /** Restarts. */ protected volatile boolean restarts; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java index 852e855c6db92..844a4300dadab 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java @@ -36,6 +36,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -55,6 +56,13 @@ public class CacheLockReleaseNodeLeaveTest extends GridCommonAbstractTest { /** */ private static final String REPLICATED_TEST_CACHE = "REPLICATED_TEST_CACHE"; + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPrimarySyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPrimarySyncSelfTest.java index 625cb18a3d792..f6b9302896b13 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPrimarySyncSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPrimarySyncSelfTest.java @@ -24,6 +24,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -70,6 +71,9 @@ public abstract class GridCacheAbstractPrimarySyncSelfTest extends GridCommonAbs /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + if (nearConfiguration() != null) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + assert GRID_CNT > 1; startGrids(GRID_CNT); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheBasicOpAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheBasicOpAbstractTest.java index a55ff2d2ef5f7..039881315ce22 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheBasicOpAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheBasicOpAbstractTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -73,6 +74,11 @@ public abstract class GridCacheBasicOpAbstractTest extends GridCommonAbstractTes /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + startGridsMultiThreaded(3); ignite1 = grid(0); @@ -327,6 +333,8 @@ public void testOptimisticTransaction() throws Exception { * @throws Exception In case of error. */ public void testPutWithExpiration() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); IgniteCache cache2 = ignite2.cache(DEFAULT_CACHE_NAME); IgniteCache cache3 = ignite3.cache(DEFAULT_CACHE_NAME); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheLockAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheLockAbstractTest.java index 16c1f5a8cce87..e6335900b3de6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheLockAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheLockAbstractTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestThread; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -65,6 +66,13 @@ public abstract class GridCacheLockAbstractTest extends GridCommonAbstractTest { /** Ip-finder. */ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java index 1d56ab6cbc00c..912aecea96c8a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -87,6 +88,8 @@ public abstract class GridCacheMultiNodeAbstractTest extends GridCommonAbstractT /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + ignite1 = startGrid(1); ignite2 = startGrid(2); ignite3 = startGrid(3); @@ -228,6 +231,8 @@ private void checkPuts(int cnt, Ignite... ignites) throws Exception { * @throws Exception If test failed. */ public void testLockUnlock() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + CacheEventListener lockLsnr1 = new CacheEventListener(ignite1, new CountDownLatch(1), EVT_CACHE_OBJECT_LOCKED); addListener(ignite1, lockLsnr1, EVT_CACHE_OBJECT_LOCKED); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeLockAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeLockAbstractTest.java index cc3b894cb41d0..6636196233e7f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeLockAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeLockAbstractTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestThread; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -63,6 +64,14 @@ public abstract class GridCacheMultiNodeLockAbstractTest extends GridCommonAbstr /** Listeners. */ private static Collection> lsnrs = new ArrayList<>(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java index 8de2d79b097fd..6375e3efd254b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -75,6 +76,13 @@ public abstract class GridCacheNodeFailureAbstractTest extends GridCommonAbstrac /** Grid instances. */ private static final List IGNITEs = new ArrayList<>(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.setUp(); + } + /** * Start grid by default. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java index d25304beda669..801495f610cd9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java @@ -55,6 +55,13 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract /** */ private int backupCnt; + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -100,10 +107,7 @@ public class GridCachePartitionNotLoadedEventSelfTest extends GridCommonAbstract public void testPrimaryAndBackupDead() throws Exception { backupCnt = 1; - startGrid(0); - startGrid(1); - startGrid(2); - startGrid(3); + startGridsMultiThreaded(4); awaitPartitionMapExchange(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest.java new file mode 100644 index 0000000000000..cf8115c52213f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMvccTxMultiThreadedSelfTest; + +/** + * + */ +public class GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest + extends GridCachePartitionedMvccTxMultiThreadedSelfTest { + /** {@inheritDoc} */ + @Override protected boolean nearEnabled() { + return false; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadRestartAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadRestartAbstractSelfTest.java index 2ea1ed81af7e4..8ef5fb8791c20 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadRestartAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadRestartAbstractSelfTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -96,6 +97,14 @@ public abstract class GridCachePreloadRestartAbstractSelfTest extends GridCommon /** */ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (nearEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); @@ -197,6 +206,9 @@ public void testAsyncPreloadRestart() throws Exception { * @throws Exception If failed. */ public void testDisabledPreloadRestart() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10261"); + preloadMode = NONE; checkRestart(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTransformEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTransformEventSelfTest.java index 6f27ce728da73..99f397e91d1c9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTransformEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTransformEventSelfTest.java @@ -49,6 +49,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -74,6 +75,9 @@ public class GridCacheTransformEventSelfTest extends GridCommonAbstractTest { /** Cache name. */ private static final String CACHE_NAME = "cache"; + /** IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** Key 1. */ private Integer key1; @@ -83,9 +87,6 @@ public class GridCacheTransformEventSelfTest extends GridCommonAbstractTest { /** Two keys in form of a set. */ private Set keys; - /** IP finder. */ - private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - /** Nodes. */ private Ignite[] ignites; @@ -327,6 +328,17 @@ public void testTxLocalPessimisticSerializable() throws Exception { checkTx(LOCAL, PESSIMISTIC, SERIALIZABLE); } + /** + * Test TRANSACTIONAL_SNAPSHOT LOCAL cache with PESSIMISTIC/REPEATABLE_READ transaction. + * + * @throws Exception If failed. + */ + public void testMvccTxLocalPessimisticRepeatableRead() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + checkMvccTx(LOCAL, PESSIMISTIC, REPEATABLE_READ); + } + /** * Test TRANSACTIONAL PARTITIONED cache with OPTIMISTIC/REPEATABLE_READ transaction. * @@ -381,6 +393,18 @@ public void testTxPartitionedPessimisticSerializable() throws Exception { checkTx(PARTITIONED, PESSIMISTIC, SERIALIZABLE); } + /** + * Test TRANSACTIONAL_SNAPSHOT PARTITIONED cache with PESSIMISTIC/REPEATABLE_READ transaction. + * + * @throws Exception If failed. + */ + public void testMvccTxPartitionedPessimisticRepeatableRead() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9321"); + + checkMvccTx(PARTITIONED, PESSIMISTIC, REPEATABLE_READ); + } + + /** * Test TRANSACTIONAL REPLICATED cache with OPTIMISTIC/REPEATABLE_READ transaction. * @@ -435,6 +459,17 @@ public void testTxReplicatedPessimisticSerializable() throws Exception { checkTx(REPLICATED, PESSIMISTIC, SERIALIZABLE); } + /** + * Test TRANSACTIONAL_SNAPSHOT REPLICATED cache with PESSIMISTIC/REPEATABLE_READ transaction. + * + * @throws Exception If failed. + */ + public void testMvccTxReplicatedPessimisticRepeatableRead() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9321"); + + checkMvccTx(REPLICATED, PESSIMISTIC, REPEATABLE_READ); + } + /** * Test ATOMIC LOCAL cache. * @@ -494,6 +529,21 @@ private void checkAtomic(CacheMode cacheMode) throws Exception { checkEventNodeIdsStrict(TransformerWithInjection.class.getName(), primaryIdsForKeys(key1, key2)); } + /** + * Check TRANSACTIONAL_SNAPSHOT cache. + * + * @param cacheMode Cache mode. + * @param txConcurrency TX concurrency. + * @param txIsolation TX isolation. + * @throws Exception If failed. + */ + private void checkMvccTx(CacheMode cacheMode, TransactionConcurrency txConcurrency, + TransactionIsolation txIsolation) throws Exception { + initialize(cacheMode, TRANSACTIONAL_SNAPSHOT, txConcurrency, txIsolation); + + checkTx0(); + } + /** * Check TRANSACTIONAL cache. * @@ -506,6 +556,14 @@ private void checkTx(CacheMode cacheMode, TransactionConcurrency txConcurrency, TransactionIsolation txIsolation) throws Exception { initialize(cacheMode, TRANSACTIONAL, txConcurrency, txIsolation); + checkTx0(); + } + + /** + * Check TX cache. + */ + private void checkTx0() { + System.out.println("BEFORE: " + evts.size()); caches[0].invoke(key1, new Transformer()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java index 300ecb938f42f..655f9971eccd2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodeChangingTopologyTest.java @@ -85,6 +85,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -139,6 +140,13 @@ public class IgniteCacheClientNodeChangingTopologyTest extends GridCommonAbstrac return cfg; } + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java index d5212e5e98547..acb800ac54d7a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java @@ -31,6 +31,8 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.testframework.MvccFeatureChecker; +import org.apache.ignite.testframework.MvccFeatureChecker.Feature; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -107,14 +109,14 @@ public void checkModes(int gridCnt) throws Exception { try { for (CacheMode mode : CacheMode.values()) { for (CacheAtomicityMode atomMode : CacheAtomicityMode.values()) { - if (mode == CacheMode.PARTITIONED && atomMode != CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) { + if (mode == CacheMode.PARTITIONED) { // Near cache makes sense only for partitioned cache. checkTxCache(CacheMode.PARTITIONED, atomMode, true, false); } - checkTxCache(CacheMode.PARTITIONED, atomMode, false, true); + checkTxCache(mode, atomMode, false, true); - checkTxCache(CacheMode.PARTITIONED, atomMode, false, false); + checkTxCache(mode, atomMode, false, false); } } } @@ -132,6 +134,13 @@ private void checkTxCache( boolean nearEnabled, boolean useEvicPlc ) throws Exception { + if (atomMode == CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) { + if (!MvccFeatureChecker.isSupported(mode) || + (nearEnabled && !MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) || + (useEvicPlc && !MvccFeatureChecker.isSupported(Feature.EVICTION))) + return; // Nothing to do. Mode is not supported. + } + final Ignite ignite = grid(0); final CacheConfiguration ccfg = cacheConfiguration( @@ -156,14 +165,11 @@ private void checkTxCache( for (TransactionIsolation iso : TransactionIsolation.values()) { for (TransactionConcurrency con : TransactionConcurrency.values()) { - try (Transaction transaction = ignite.transactions().txStart(con, iso)) { - //TODO: IGNITE-7187: Fix when ticket will be implemented. (Near cache) - //TODO: IGNITE-7956: Fix when ticket will be implemented. (Eviction) - if (((IgniteCacheProxy)cache).context().mvccEnabled() && - ((iso != TransactionIsolation.REPEATABLE_READ && con != TransactionConcurrency.PESSIMISTIC) - || nearEnabled || useEvicPlc)) - return; // Nothing to do. Mode is not supported. + if (atomMode == CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT && + !MvccFeatureChecker.isSupported(con, iso)) + continue; // Mode not supported. + try (Transaction transaction = ignite.transactions().txStart(con, iso)) { assertEquals(val, cache.get(key)); transaction.commit(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteMvccTxTimeoutAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteMvccTxTimeoutAbstractTest.java new file mode 100644 index 0000000000000..15d23f4d4c64a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteMvccTxTimeoutAbstractTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.Random; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; +import org.apache.ignite.transactions.TransactionTimeoutException; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Simple cache test. + */ +public class IgniteMvccTxTimeoutAbstractTest extends GridCommonAbstractTest { + /** Random number generator. */ + private static final Random RAND = new Random(); + + /** Grid count. */ + private static final int GRID_COUNT = 2; + + /** Transaction timeout. */ + private static final long TIMEOUT = 50; + + /** */ + private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** + * @throws Exception If failed. + */ + @Override protected void beforeTestsStarted() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7388"); + + startGridsMultiThreaded(GRID_COUNT, true); + } + + /** + * @throws Exception If failed. + */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration c = super.getConfiguration(igniteInstanceName); + + TransactionConfiguration txCfg = c.getTransactionConfiguration(); + + txCfg.setDefaultTxTimeout(TIMEOUT); + + TcpDiscoverySpi spi = new TcpDiscoverySpi(); + + spi.setIpFinder(ipFinder); + + c.setDiscoverySpi(spi); + + return c; + } + + /** + * @param i Grid index. + * @return Cache. + */ + @Override protected IgniteCache jcache(int i) { + return grid(i).cache(DEFAULT_CACHE_NAME); + } + + /** + * @throws IgniteCheckedException If test failed. + */ + public void testPessimisticRepeatableRead() throws Exception { + checkTransactionTimeout(PESSIMISTIC, REPEATABLE_READ); + } + + /** + * @param concurrency Concurrency. + * @param isolation Isolation. + * @throws IgniteCheckedException If test failed. + */ + private void checkTransactionTimeout(TransactionConcurrency concurrency, + TransactionIsolation isolation) throws Exception { + int idx = RAND.nextInt(GRID_COUNT); + + IgniteCache cache = jcache(idx); + + Transaction tx = ignite(idx).transactions().txStart(concurrency, isolation, TIMEOUT, 0); + + try { + info("Storing value in cache [key=1, val=1]"); + + cache.put(1, "1"); + + long sleep = TIMEOUT * 2; + + info("Going to sleep for (ms): " + sleep); + + Thread.sleep(sleep); + + info("Storing value in cache [key=1, val=2]"); + + cache.put(1, "2"); + + info("Committing transaction: " + tx); + + tx.commit(); + + assert false : "Timeout never happened for transaction: " + tx; + } + catch (Exception e) { + if (!(X.hasCause(e, TransactionTimeoutException.class))) + throw e; + + info("Received expected timeout exception [msg=" + e.getMessage() + ", tx=" + tx + ']'); + } + finally { + tx.close(); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxTimeoutAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxTimeoutAbstractTest.java index a0ec70a9c3519..4cd64e3a41638 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxTimeoutAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxTimeoutAbstractTest.java @@ -17,13 +17,15 @@ package org.apache.ignite.internal.processors.cache.distributed; -import java.util.ArrayList; -import java.util.List; import java.util.Random; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -46,25 +48,41 @@ public class IgniteTxTimeoutAbstractTest extends GridCommonAbstractTest { /** Grid count. */ private static final int GRID_COUNT = 2; - /** Grid instances. */ - private static final List IGNITEs = new ArrayList<>(); - /** Transaction timeout. */ private static final long TIMEOUT = 50; + /** */ + private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** * @throws Exception If failed. */ @Override protected void beforeTestsStarted() throws Exception { - for (int i = 0; i < GRID_COUNT; i++) - IGNITEs.add(startGrid(i)); + startGridsMultiThreaded(GRID_COUNT); } /** * @throws Exception If failed. */ @Override protected void afterTestsStopped() throws Exception { - IGNITEs.clear(); + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration c = super.getConfiguration(igniteInstanceName); + + TransactionConfiguration txCfg = c.getTransactionConfiguration(); + + txCfg.setDefaultTxTimeout(TIMEOUT); + + TcpDiscoverySpi spi = new TcpDiscoverySpi(); + + spi.setIpFinder(ipFinder); + + c.setDiscoverySpi(spi); + + return c; } /** @@ -72,7 +90,7 @@ public class IgniteTxTimeoutAbstractTest extends GridCommonAbstractTest { * @return Cache. */ @Override protected IgniteCache jcache(int i) { - return IGNITEs.get(i).cache(DEFAULT_CACHE_NAME); + return grid(i).cache(DEFAULT_CACHE_NAME); } /** @@ -124,7 +142,6 @@ public void testOptimisticSerializable() throws Exception { */ private void checkTransactionTimeout(TransactionConcurrency concurrency, TransactionIsolation isolation) throws Exception { - int idx = RAND.nextInt(GRID_COUNT); IgniteCache cache = jcache(idx); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheColocatedMvccTxSingleThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheColocatedMvccTxSingleThreadedSelfTest.java new file mode 100644 index 0000000000000..717d77113413f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheColocatedMvccTxSingleThreadedSelfTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.IgniteMvccTxSingleThreadedAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.NONE; + +/** + * Test Mvcc txs in single-threaded mode for colocated cache. + */ +public class GridCacheColocatedMvccTxSingleThreadedSelfTest extends IgniteMvccTxSingleThreadedAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setCacheMode(PARTITIONED); + ccfg.setBackups(1); + ccfg.setNearConfiguration(null); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + ccfg.setEvictionPolicy(null); + + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_ASYNC); + + ccfg.setRebalanceMode(NONE); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 4; + } + + /** {@inheritDoc} */ + @Override protected int keyCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int maxKeyValue() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int iterations() { + return 3000; + } + + /** {@inheritDoc} */ + @Override protected boolean isTestDebug() { + return false; + } + + /** {@inheritDoc} */ + @Override protected boolean printMemoryStats() { + return true; + } + +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEntrySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEntrySelfTest.java index 7767da9721017..1ac8216702604 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEntrySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtEntrySelfTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -78,6 +79,10 @@ public class GridCacheDhtEntrySelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + startGridsMultiThreaded(GRID_CNT); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtMappingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtMappingSelfTest.java index bd0af345fed34..5456e4ca17556 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtMappingSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtMappingSelfTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -43,6 +44,13 @@ public class GridCacheDhtMappingSelfTest extends GridCommonAbstractTest { /** IP finder. */ private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java index 66efc18fc0a6b..d8bc44a850508 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -73,6 +74,13 @@ public class GridCacheDhtPreloadDisabledSelfTest extends GridCommonAbstractTest /** IP finder. */ private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java index 6c32a67b98f24..5f7be7afb1cce 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -137,6 +138,8 @@ public void testConcurrentNodesStart() throws Exception { * @throws Exception If failed. */ public void testConcurrentNodesStartStop() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + try { multithreadedAsync( new Callable() { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadPutGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadPutGetSelfTest.java index 71911e8f9c965..a62984a525fb9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadPutGetSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadPutGetSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -163,6 +164,9 @@ public void testPutGetNone0() throws Exception { * @throws Exception If failed. */ public void testPutGetNone1() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10261"); + preloadMode = NONE; backups = 1; @@ -173,6 +177,9 @@ public void testPutGetNone1() throws Exception { * @throws Exception If failed. */ public void testPutGetNone2() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10261"); + preloadMode = NONE; backups = 2; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java index 82ed95e52531f..7585e72190982 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java @@ -212,7 +212,6 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception */ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle) throws Exception { - try { Ignite ignite1 = startGrid(0); @@ -456,7 +455,6 @@ private void stopGrids(Iterable grids) { */ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle) throws Exception { - try { Ignite ignite1 = startGrid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheClearDuringRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheClearDuringRebalanceTest.java index 8561c5c5211b9..5194020b1a08b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheClearDuringRebalanceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheClearDuringRebalanceTest.java @@ -28,6 +28,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -42,6 +43,14 @@ public class IgniteCacheClearDuringRebalanceTest extends GridCommonAbstractTest /** */ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if(MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java index 98520ab01a0de..6cecce148967f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePartitionedBackupNodeFailureRecoveryTest.java @@ -44,7 +44,7 @@ /** */ public class IgniteCachePartitionedBackupNodeFailureRecoveryTest extends IgniteCacheAbstractTest { - /** {@inheritDoc}*/ + /** {@inheritDoc}*/ @Override protected int gridCount() { return 3; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearClientHitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearClientHitTest.java index 1dd62e49863be..8634cbb7309ab 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearClientHitTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearClientHitTest.java @@ -71,14 +71,11 @@ private CacheConfiguration cacheConfiguration() { CacheConfiguration cfg = new CacheConfiguration<>(); cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); - cfg.setCacheMode(CacheMode.PARTITIONED); - cfg.setBackups(1); - cfg.setCopyOnRead(false); - cfg.setName(CACHE_NAME); + cfg.setNearConfiguration(new NearCacheConfiguration<>()); return cfg; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiGetSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiGetSelfTest.java index e27c9ad8fe9ee..21cabe3163bef 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiGetSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiGetSelfTest.java @@ -20,12 +20,13 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.affinity.Affinity; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -35,9 +36,9 @@ import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; import org.apache.ignite.transactions.TransactionOptimisticException; -import org.apache.log4j.Level; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.NONE; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; @@ -50,9 +51,6 @@ * Test getting the same value twice within the same transaction. */ public class GridCacheNearMultiGetSelfTest extends GridCommonAbstractTest { - /** Cache debug flag. */ - private static final boolean CACHE_DEBUG = false; - /** Number of gets. */ private static final int GET_CNT = 5; @@ -62,41 +60,48 @@ public class GridCacheNearMultiGetSelfTest extends GridCommonAbstractTest { /** */ private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** */ + private CacheAtomicityMode atomicityMode; + /** {@inheritDoc} */ @SuppressWarnings({"ConstantConditions"}) @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); - c.getTransactionConfiguration().setTxSerializableEnabled(true); - - CacheConfiguration cc = defaultCacheConfiguration(); - - cc.setCacheMode(PARTITIONED); - cc.setBackups(1); - cc.setAtomicityMode(TRANSACTIONAL); - - cc.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - - cc.setRebalanceMode(NONE); - TcpDiscoverySpi spi = new TcpDiscoverySpi(); spi.setIpFinder(ipFinder); c.setDiscoverySpi(spi); - c.setCacheConfiguration(cc); + return c; + } - if (CACHE_DEBUG) - resetLog4j(Level.DEBUG, false, GridCacheProcessor.class.getPackage().getName()); + /** + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration() { + CacheConfiguration cc = defaultCacheConfiguration(); - return c; + cc.setCacheMode(PARTITIONED); + cc.setBackups(1); + cc.setAtomicityMode(atomicityMode); + cc.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cc.setRebalanceMode(NONE); + cc.setNearConfiguration(new NearCacheConfiguration()); + + return cc; } /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { - for (int i = 0; i < GRID_CNT; i++) - startGrid(i); + startGridsMultiThreaded(GRID_CNT); + } + + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + atomicityMode = TRANSACTIONAL; } /** {@inheritDoc} */ @@ -111,6 +116,8 @@ public class GridCacheNearMultiGetSelfTest extends GridCommonAbstractTest { assertEquals("Cache size mismatch for grid [igniteInstanceName=" + g.name() + ", entrySet=" + entrySet(c) + ']', 0, c.size()); } + + grid(0).destroyCache(DEFAULT_CACHE_NAME); } /** @return {@code True} if debug enabled. */ @@ -214,6 +221,24 @@ public void testPessimisticSerializable() throws Exception { checkDoubleGet(PESSIMISTIC, SERIALIZABLE, true); } + /** @throws Exception If failed. */ + public void testMvccPessimisticRepeatableReadNoPut() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + checkDoubleGet(PESSIMISTIC, REPEATABLE_READ, false); + } + + /** @throws Exception If failed. */ + public void testMvccPessimisticRepeatableReadWithPut() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + checkDoubleGet(PESSIMISTIC, REPEATABLE_READ, true); + } + /** * @param concurrency Concurrency. * @param isolation Isolation. @@ -223,7 +248,7 @@ public void testPessimisticSerializable() throws Exception { private void checkDoubleGet(TransactionConcurrency concurrency, TransactionIsolation isolation, boolean put) throws Exception { IgniteEx ignite = grid(0); - IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); + IgniteCache cache = ignite.getOrCreateCache(cacheConfiguration()); Integer key = 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java index 06e18627ab95f..c89372ea15f58 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java @@ -56,6 +56,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.Nullable; @@ -79,6 +80,15 @@ public class GridCacheNearMultiNodeSelfTest extends GridCommonAbstractTest { /** Cache store. */ private static TestStore store = new TestStore(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.setUp(); + } + /** */ private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOneNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOneNodeSelfTest.java index d4f485aaaf2f7..d25facd97c4dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOneNodeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOneNodeSelfTest.java @@ -26,10 +26,12 @@ import org.apache.ignite.cache.store.CacheStoreAdapter; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -46,6 +48,13 @@ public class GridCacheNearOneNodeSelfTest extends GridCommonAbstractTest { /** Cache store. */ private static TestStore store = new TestStore(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** * */ @@ -80,8 +89,8 @@ public GridCacheNearOneNodeSelfTest() { cacheCfg.setCacheMode(PARTITIONED); cacheCfg.setBackups(1); cacheCfg.setAtomicityMode(TRANSACTIONAL); - cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setNearConfiguration(new NearCacheConfiguration()); cacheCfg.setCacheStoreFactory(singletonFactory(store)); cacheCfg.setReadThrough(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearPartitionedClearSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearPartitionedClearSelfTest.java index cbdc8551dd7f1..90a740648a883 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearPartitionedClearSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearPartitionedClearSelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -56,6 +57,13 @@ public class GridCacheNearPartitionedClearSelfTest extends GridCommonAbstractTes /** Shared IP finder. */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { startGrids(GRID_CNT); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReaderPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReaderPreloadSelfTest.java index e143260f798d2..316cc6e75e28d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReaderPreloadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReaderPreloadSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -61,6 +62,13 @@ public class GridCacheNearReaderPreloadSelfTest extends GridCommonAbstractTest { /** Cache on backup node. */ private IgniteCache cache3; + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { cache1 = null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReadersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReadersSelfTest.java index cbb2032302ee2..9ee5114d2b4a3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReadersSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearReadersSelfTest.java @@ -48,6 +48,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; @@ -67,11 +68,18 @@ public class GridCacheNearReadersSelfTest extends GridCommonAbstractTest { private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); /** Grid counter. */ - private static AtomicInteger cntr = new AtomicInteger(0); + private AtomicInteger cntr = new AtomicInteger(0); /** Test cache affinity. */ private GridCacheModuloAffinityFunction aff = new GridCacheModuloAffinityFunction(); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedBasicStoreMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedBasicStoreMultiNodeSelfTest.java index 91c9b22b793bd..73d5b5ffd1491 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedBasicStoreMultiNodeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedBasicStoreMultiNodeSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -72,6 +73,11 @@ public class GridCachePartitionedBasicStoreMultiNodeSelfTest extends GridCommonA /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + if (nearCacheConfiguration() != null) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + super.beforeTestsStarted(); stores = Collections.synchronizedList(new ArrayList()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java index 1c79db7e7a111..54eb32d3f1ce6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.distributed.GridCacheEventAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -30,6 +31,13 @@ * Tests events. */ public class GridCachePartitionedEventSelfTest extends GridCacheEventAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { CacheConfiguration cfg = defaultCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java index 96fb8f65bb56c..28afce1e9fe8e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -50,6 +51,13 @@ public class GridCachePartitionedExplicitLockNodeFailureSelfTest extends GridCom /** */ public static final int GRID_CNT = 4; + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLoadCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLoadCacheSelfTest.java index 94749509511da..2ad8b50aad329 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLoadCacheSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLoadCacheSelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -48,6 +49,13 @@ public class GridCachePartitionedLoadCacheSelfTest extends GridCommonAbstractTes /** Puts count. */ private static final int PUT_CNT = 100; + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxMultiThreadedSelfTest.java new file mode 100644 index 0000000000000..184132ef5f16e --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxMultiThreadedSelfTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.near; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.processors.cache.IgniteMvccTxMultiThreadedAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; + +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * Tests for partitioned cache transactions. + */ +public class GridCachePartitionedMvccTxMultiThreadedSelfTest extends IgniteMvccTxMultiThreadedAbstractTest { + @Override protected void beforeTestsStarted() throws Exception { + if (nearEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setCacheMode(PARTITIONED); + ccfg.setBackups(1); + + ccfg.setEvictionPolicy(null); + + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + ccfg.setNearConfiguration(nearEnabled() ? new NearCacheConfiguration() : null); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** + * @return {@code True} if near cache is enabled. + */ + protected boolean nearEnabled() { + return true; + } + + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int keyCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int maxKeyValue() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int threadCount() { + return 5; + } + + /** {@inheritDoc} */ + @Override protected int iterations() { + return 1000; + } + + /** {@inheritDoc} */ + @Override protected boolean isTestDebug() { + return false; + } + + /** {@inheritDoc} */ + @Override protected boolean printMemoryStats() { + return true; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxSingleThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxSingleThreadedSelfTest.java new file mode 100644 index 0000000000000..140483912d894 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxSingleThreadedSelfTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.near; + +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.IgniteMvccTxSingleThreadedAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.NONE; + +/** + * Tests for partitioned cache transactions. + */ +public class GridCachePartitionedMvccTxSingleThreadedSelfTest extends IgniteMvccTxSingleThreadedAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setCacheMode(PARTITIONED); + ccfg.setBackups(1); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + ccfg.setNearConfiguration(null); + ccfg.setEvictionPolicy(null); + + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_ASYNC); + + ccfg.setRebalanceMode(NONE); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 4; + } + + /** {@inheritDoc} */ + @Override protected int keyCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int maxKeyValue() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int iterations() { + return 3000; + } + + /** {@inheritDoc} */ + @Override protected boolean isTestDebug() { + return false; + } + + /** {@inheritDoc} */ + @Override protected boolean printMemoryStats() { + return true; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxTimeoutSelfTest.java new file mode 100644 index 0000000000000..aad95aab29bff --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMvccTxTimeoutSelfTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.near; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.distributed.IgniteMvccTxTimeoutAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; + +/** + * Simple cache test. + */ +public class GridCachePartitionedMvccTxTimeoutSelfTest extends IgniteMvccTxTimeoutAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration c = super.getConfiguration(igniteInstanceName); + + CacheConfiguration cc = defaultCacheConfiguration(); + + cc.setCacheMode(PARTITIONED); + cc.setBackups(1); + cc.setAtomicityMode(TRANSACTIONAL); + + //cacheCfg.setPreloadMode(NONE); + + c.setCacheConfiguration(cc); + + return c; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedPreloadLifecycleSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedPreloadLifecycleSelfTest.java index 3a52d64a0de0c..ea384e35beedb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedPreloadLifecycleSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedPreloadLifecycleSelfTest.java @@ -57,7 +57,6 @@ public class GridCachePartitionedPreloadLifecycleSelfTest extends GridCachePrelo cc1.setRebalanceMode(preloadMode); cc1.setEvictionPolicy(null); cc1.setCacheStoreFactory(null); - cc1.setEvictionPolicy(null); // Identical configuration. CacheConfiguration cc2 = new CacheConfiguration(cc1); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxTimeoutSelfTest.java index 202476e160919..1a88ae0f77936 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxTimeoutSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxTimeoutSelfTest.java @@ -19,11 +19,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxTimeoutAbstractTest; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -32,27 +28,10 @@ * Simple cache test. */ public class GridCachePartitionedTxTimeoutSelfTest extends IgniteTxTimeoutAbstractTest { - /** Transaction timeout. */ - private static final long TIMEOUT = 50; - - /** */ - private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); - TransactionConfiguration txCfg = c.getTransactionConfiguration(); - - txCfg.setTxSerializableEnabled(true); - txCfg.setDefaultTxTimeout(TIMEOUT); - - TcpDiscoverySpi spi = new TcpDiscoverySpi(); - - spi.setIpFinder(ipFinder); - - c.setDiscoverySpi(spi); - CacheConfiguration cc = defaultCacheConfiguration(); cc.setCacheMode(PARTITIONED); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java index bf8ad78128745..02a9223f2c012 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheStoreUpdateTest.java @@ -39,6 +39,7 @@ import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -59,6 +60,14 @@ public class GridNearCacheStoreUpdateTest extends GridCommonAbstractTest { /** */ private IgniteCache cache; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception { final IgniteConfiguration cfg = super.getConfiguration(gridName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java index d25301337b7ad..e1ca01f8ea2ac 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NearCacheSyncUpdateTest.java @@ -34,6 +34,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -60,6 +61,13 @@ public class NearCacheSyncUpdateTest extends GridCommonAbstractTest { startGridsMultiThreaded(3); } + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + /** * @throws Exception If failed. */ @@ -74,6 +82,15 @@ public void testNearCacheSyncUpdateTx() throws Exception { nearCacheSyncUpdateTx(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testNearCacheSyncUpdateMvccTx() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + nearCacheSyncUpdateTx(TRANSACTIONAL_SNAPSHOT); + } + /** * @param atomicityMode Atomicity mode. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java index 5d82f832ba920..30618821c40b5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java @@ -46,9 +46,18 @@ public class NoneRebalanceModeSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + startGrid(0); } + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxTimeoutSelfTest.java index a63a302692028..9a748ba30543d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxTimeoutSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxTimeoutSelfTest.java @@ -20,9 +20,6 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxTimeoutAbstractTest; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -30,31 +27,16 @@ * Simple cache test. */ public class GridCacheReplicatedTxTimeoutSelfTest extends IgniteTxTimeoutAbstractTest { - /** Transaction timeout. */ - private static final long TIMEOUT = 50; - - /** */ - private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); - c.getTransactionConfiguration().setDefaultTxTimeout(TIMEOUT); - c.getTransactionConfiguration().setTxSerializableEnabled(true); - CacheConfiguration cc = defaultCacheConfiguration(); cc.setCacheMode(REPLICATED); c.setCacheConfiguration(cc); - TcpDiscoverySpi spi = new TcpDiscoverySpi(); - - spi.setIpFinder(ipFinder); - - c.setDiscoverySpi(spi); - return c; } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicApiSelfTest.java index d35590f07739d..91df59911517f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicApiSelfTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheBasicApiAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -27,6 +28,13 @@ * Basic API tests. */ public class GridCacheLocalBasicApiSelfTest extends GridCacheBasicApiAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicStoreSelfTest.java index 183b456c71b00..5d6492b57eee7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalBasicStoreSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.GridCacheBasicStoreAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -26,6 +27,13 @@ * Test store with local cache. */ public class GridCacheLocalBasicStoreSelfTest extends GridCacheBasicStoreAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java index 0bf01c2f26732..c0b60718001db 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.distributed.GridCacheEventAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -26,6 +27,14 @@ * Tests events. */ public class GridCacheLocalEventSelfTest extends GridCacheEventAbstractTest { + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEvictionEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEvictionEventSelfTest.java index b2d98c61c1150..74066899ddfcd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEvictionEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEvictionEventSelfTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.GridCacheEvictionEventAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -27,6 +28,13 @@ * Tests local cache eviction event. */ public class GridCacheLocalEvictionEventSelfTest extends GridCacheEvictionEventAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return CacheMode.LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalGetAndTransformStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalGetAndTransformStoreSelfTest.java index 19b4b6116be2f..9752b7eae480a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalGetAndTransformStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalGetAndTransformStoreSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.GridCacheGetAndTransformStoreAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -26,6 +27,13 @@ * Test get and transform for store with local cache. */ public class GridCacheLocalGetAndTransformStoreSelfTest extends GridCacheGetAndTransformStoreAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIsolatedNodesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIsolatedNodesSelfTest.java index 53122c78f5391..e5d3b485c31bb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIsolatedNodesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIsolatedNodesSelfTest.java @@ -23,6 +23,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -31,6 +32,13 @@ * Isolated nodes LOCAL cache self test. */ public class GridCacheLocalIsolatedNodesSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLoadAllSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLoadAllSelfTest.java index f10cefdb5ede9..647027fc3fc27 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLoadAllSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLoadAllSelfTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -35,6 +36,13 @@ * Load-All self test. */ public class GridCacheLocalLoadAllSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLockSelfTest.java index f4160edba4aec..a2b74cf5c1d06 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLockSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalLockSelfTest.java @@ -28,6 +28,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestThread; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -38,6 +39,13 @@ */ @SuppressWarnings({"ProhibitedExceptionThrown"}) public class GridCacheLocalLockSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** Grid. */ private Ignite ignite; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMultithreadedSelfTest.java index f6dc5351892a5..491163158028b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMultithreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMultithreadedSelfTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestThread; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -42,6 +43,13 @@ * Multithreaded local cache locking test. */ public class GridCacheLocalMultithreadedSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** Cache. */ private IgniteCache cache; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java index 388a7bfb78958..816168f0a0007 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.IgniteTxMultiThreadedAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -33,6 +34,13 @@ public class GridCacheLocalTxMultiThreadedSelfTest extends IgniteTxMultiThreaded /** Cache debug flag. */ private static final boolean CACHE_DEBUG = false; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @SuppressWarnings({"ConstantConditions"}) @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxSingleThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxSingleThreadedSelfTest.java index 83c74e201abd2..c2c3f813cc3fc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxSingleThreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxSingleThreadedSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.IgniteTxSingleThreadedAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -30,6 +31,13 @@ * Tests for local transactions. */ public class GridCacheLocalTxSingleThreadedSelfTest extends IgniteTxSingleThreadedAbstractTest { + /** {@inheritDoc} */ + @Override public void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** Cache debug flag. */ private static final boolean CACHE_DEBUG = false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxTimeoutSelfTest.java index 160e2512b180c..62af45242c052 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxTimeoutSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxTimeoutSelfTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -42,6 +43,13 @@ * */ public class GridCacheLocalTxTimeoutSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** Grid. */ private Ignite ignite; diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java new file mode 100644 index 0000000000000..ee9b02af08e00 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testframework; + +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS; +import static org.junit.Assert.fail; + +/** + * Provides checks for features supported when FORCE_MVCC mode is on. + */ +public class MvccFeatureChecker { + /** */ + private static final boolean FORCE_MVCC = + IgniteSystemProperties.getBoolean(IGNITE_FORCE_MVCC_MODE_IN_TESTS, false); + + /** */ + public enum Feature { + CACHE_STORE, + NEAR_CACHE, + LOCAL_CACHE, + ENTRY_LOCK, + CACHE_EVENTS, + EVICTION, + EXPIRATION + } + + /** + * Fails if feature is not supported. + * + * @param f feature. + * @throws AssertionError If failed. + */ + public static void failIfNotSupported(Feature f) { + if (!forcedMvcc()) + return; + + validateFeature(f); + } + + /** + * @return {@code True} if Mvcc mode is forced. + */ + public static boolean forcedMvcc() { + return FORCE_MVCC; + } + + /** + * Check if feature is supported. + * + * @param f Feature. + * @return {@code True} if feature is supported, {@code False} otherwise. + */ + public static boolean isSupported(Feature f) { + try { + validateFeature(f); + + return true; + } + catch (AssertionError ignore) { + return false; + } + } + + /** + * Check if Tx mode is supported. + * + * @param conc Transaction concurrency. + * @param iso Transaction isolation. + * @return {@code True} if feature is supported, {@code False} otherwise. + */ + public static boolean isSupported(TransactionConcurrency conc, TransactionIsolation iso) { + return conc == TransactionConcurrency.PESSIMISTIC && + iso == TransactionIsolation.REPEATABLE_READ; + } + + + /** + * Check if Cache mode is supported. + * + * @param mode Cache mode. + * @return {@code True} if feature is supported, {@code False} otherwise. + */ + public static boolean isSupported(CacheMode mode) { + return mode != CacheMode.LOCAL || isSupported(Feature.LOCAL_CACHE); + } + + /** + * Fails if feature is not supported in Mvcc mode. + * + * @param feature Mvcc feature. + * @throws AssertionError If failed. + */ + private static void validateFeature(Feature feature) { + switch (feature) { + case NEAR_CACHE: + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + case LOCAL_CACHE: + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + case CACHE_STORE: + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + case ENTRY_LOCK: + fail("https://issues.apache.org/jira/browse/IGNITE-9324"); + + case CACHE_EVENTS: + fail("https://issues.apache.org/jira/browse/IGNITE-9321"); + + case EVICTION: + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + case EXPIRATION: + fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java new file mode 100644 index 0000000000000..e0d1eef95c18e --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.affinity.rendezvous.ClusterNodeAttributeAffinityBackupFilterSelfTest; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionExcludeNeighborsSelfTest; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionSelfTest; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionStandardHashSelfTest; +import org.apache.ignite.internal.IgniteReflectionFactorySelfTest; +import org.apache.ignite.internal.processors.cache.CacheComparatorTest; +import org.apache.ignite.internal.processors.cache.CacheConfigurationLeakTest; +import org.apache.ignite.internal.processors.cache.CacheEnumOperationsSingleNodeTest; +import org.apache.ignite.internal.processors.cache.CacheEnumOperationsTest; +import org.apache.ignite.internal.processors.cache.CacheExchangeMessageDuplicatedStateTest; +import org.apache.ignite.internal.processors.cache.CacheGroupLocalConfigurationSelfTest; +import org.apache.ignite.internal.processors.cache.CacheOptimisticTransactionsWithFilterSingleServerTest; +import org.apache.ignite.internal.processors.cache.CacheOptimisticTransactionsWithFilterTest; +import org.apache.ignite.internal.processors.cache.GridCacheAtomicMessageCountSelfTest; +import org.apache.ignite.internal.processors.cache.GridCachePartitionedProjectionAffinitySelfTest; +import org.apache.ignite.internal.processors.cache.IgniteAtomicCacheEntryProcessorNodeJoinTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheNoSyncForGetTest; +import org.apache.ignite.internal.processors.cache.IgniteCachePartitionMapUpdateTest; +import org.apache.ignite.internal.processors.cache.IgniteClientCacheStartFailoverTest; +import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheAndNodeStop; +import org.apache.ignite.internal.processors.cache.IgniteNearClientCacheCloseTest; +import org.apache.ignite.internal.processors.cache.IgniteOnePhaseCommitInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteOnePhaseCommitNearReadersTest; +import org.apache.ignite.internal.processors.cache.MemoryPolicyConfigValidationTest; +import org.apache.ignite.internal.processors.cache.NonAffinityCoordinatorDynamicStartStopTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheLoadingConcurrentGridStartSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheLoadingConcurrentGridStartSelfTestAllowOverwrite; +import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest; +import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionedNearDisabledTxMultiThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.GridCacheTransformEventSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheClientNodePartitionsExchangeTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheServerNodeConcurrentStart; +import org.apache.ignite.internal.processors.cache.distributed.dht.CachePartitionPartialCountersMapSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedMvccTxSingleThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedOptimisticTransactionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedTxSingleThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtAtomicEvictionNearReadersSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadOnheapSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedUnloadEventsSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedBackupNodeFailureRecoveryTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearEvictionEventSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearMultiNodeSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearReadersSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearClientHitTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearJobExecutionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearMultiGetSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxForceKeyTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedAffinitySelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedAtomicGetAndTransformStoreSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiThreadedPutGetSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMvccTxMultiThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMvccTxSingleThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMvccTxTimeoutSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxMultiThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxSingleThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxTimeoutSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheRendezvousAffinityClientSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.NearCacheSyncUpdateTest; +import org.apache.ignite.internal.processors.cache.distributed.near.NoneRebalanceModeSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedJobExecutionTest; +import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicBasicStoreSelfTest; +import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicGetAndTransformStoreSelfTest; +import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest; +import org.apache.ignite.internal.processors.continuous.IgniteNoCustomEventsOnNodeStart; + +/** + * Test suite. + */ +public class IgniteCacheMvccTestSuite2 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + HashSet ignoredTests = new HashSet<>(128); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(GridCacheTransformEventSelfTest.class); + ignoredTests.add(IgniteClientCacheStartFailoverTest.class); + ignoredTests.add(IgniteNearClientCacheCloseTest.class); + ignoredTests.add(IgniteCacheNoSyncForGetTest.class); + ignoredTests.add(CacheEnumOperationsSingleNodeTest.class); + ignoredTests.add(CacheEnumOperationsTest.class); + ignoredTests.add(NearCacheSyncUpdateTest.class); + ignoredTests.add(GridCacheNearMultiGetSelfTest.class); + + // Optimistic tx tests. + ignoredTests.add(GridCacheColocatedOptimisticTransactionSelfTest.class); + ignoredTests.add(CacheOptimisticTransactionsWithFilterSingleServerTest.class); + ignoredTests.add(CacheOptimisticTransactionsWithFilterTest.class); + + // Irrelevant Tx tests. + ignoredTests.add(IgniteOnePhaseCommitInvokeTest.class); + ignoredTests.add(IgniteOnePhaseCommitNearReadersTest.class); + ignoredTests.add(GridCacheDhtPreloadOnheapSelfTest.class); + ignoredTests.add(GridCachePartitionedMultiThreadedPutGetSelfTest.class); // On-heap test. + + // Atomic cache tests. + ignoredTests.add(GridCacheLocalAtomicBasicStoreSelfTest.class); + ignoredTests.add(GridCacheLocalAtomicGetAndTransformStoreSelfTest.class); + ignoredTests.add(GridCacheAtomicNearMultiNodeSelfTest.class); + ignoredTests.add(GridCacheAtomicNearReadersSelfTest.class); + ignoredTests.add(GridCachePartitionedAtomicGetAndTransformStoreSelfTest.class); + ignoredTests.add(GridCacheAtomicNearEvictionEventSelfTest.class); + ignoredTests.add(GridCacheAtomicMessageCountSelfTest.class); + ignoredTests.add(IgniteAtomicCacheEntryProcessorNodeJoinTest.class); + ignoredTests.add(GridCacheDhtAtomicEvictionNearReadersSelfTest.class); + ignoredTests.add(GridCacheNearClientHitTest.class); + ignoredTests.add(GridCacheNearTxForceKeyTest.class); + ignoredTests.add(CacheLoadingConcurrentGridStartSelfTest.class); + ignoredTests.add(CacheLoadingConcurrentGridStartSelfTestAllowOverwrite.class); + ignoredTests.add(IgniteCachePartitionedBackupNodeFailureRecoveryTest.class); + + // Other non-tx tests. + ignoredTests.add(RendezvousAffinityFunctionSelfTest.class); + ignoredTests.add(RendezvousAffinityFunctionExcludeNeighborsSelfTest.class); + ignoredTests.add(RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.class); + ignoredTests.add(RendezvousAffinityFunctionStandardHashSelfTest.class); + ignoredTests.add(GridCachePartitionedAffinitySelfTest.class); + ignoredTests.add(GridCacheRendezvousAffinityClientSelfTest.class); + ignoredTests.add(GridCachePartitionedProjectionAffinitySelfTest.class); + ignoredTests.add(RendezvousAffinityFunctionBackupFilterSelfTest.class); + ignoredTests.add(ClusterNodeAttributeAffinityBackupFilterSelfTest.class); + ignoredTests.add(NonAffinityCoordinatorDynamicStartStopTest.class); + + ignoredTests.add(NoneRebalanceModeSelfTest.class); + ignoredTests.add(IgniteCachePartitionMapUpdateTest.class); + ignoredTests.add(IgniteCacheClientNodePartitionsExchangeTest.class); + ignoredTests.add(IgniteCacheServerNodeConcurrentStart.class); + + ignoredTests.add(GridCachePartitionedUnloadEventsSelfTest.class); + + ignoredTests.add(IgniteNoCustomEventsOnNodeStart.class); + ignoredTests.add(CacheExchangeMessageDuplicatedStateTest.class); + ignoredTests.add(IgniteDynamicCacheAndNodeStop.class); + + ignoredTests.add(GridCacheReplicatedJobExecutionTest.class); + ignoredTests.add(GridCacheNearJobExecutionSelfTest.class); + + ignoredTests.add(CacheConfigurationLeakTest.class); + ignoredTests.add(MemoryPolicyConfigValidationTest.class); + ignoredTests.add(MemoryPolicyInitializationTest.class); + ignoredTests.add(CacheGroupLocalConfigurationSelfTest.class); + + ignoredTests.add(CachePartitionStateTest.class); + ignoredTests.add(CacheComparatorTest.class); + ignoredTests.add(CachePartitionPartialCountersMapSelfTest.class); + ignoredTests.add(IgniteReflectionFactorySelfTest.class); + + // Skip classes which Mvcc implementations are added in this method below. + // TODO IGNITE-10175: refactor these tests (use assume) to support both mvcc and non-mvcc modes after moving to JUnit4/5. + ignoredTests.add(GridCachePartitionedTxSingleThreadedSelfTest.class); // See GridCachePartitionedMvccTxSingleThreadedSelfTest + ignoredTests.add(GridCacheColocatedTxSingleThreadedSelfTest.class); // See GridCacheColocatedMvccTxSingleThreadedSelfTest + ignoredTests.add(GridCachePartitionedTxMultiThreadedSelfTest.class); // See GridCachePartitionedMvccTxMultiThreadedSelfTest + ignoredTests.add(GridCachePartitionedNearDisabledTxMultiThreadedSelfTest.class); // See GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest + ignoredTests.add(GridCachePartitionedTxTimeoutSelfTest.class); // See GridCachePartitionedMvccTxTimeoutSelfTest + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 2"); + + suite.addTest(IgniteCacheTestSuite2.suite(ignoredTests)); + + // Add Mvcc clones. + suite.addTestSuite(GridCachePartitionedMvccTxSingleThreadedSelfTest.class); + suite.addTestSuite(GridCacheColocatedMvccTxSingleThreadedSelfTest.class); + suite.addTestSuite(GridCachePartitionedMvccTxMultiThreadedSelfTest.class); + suite.addTestSuite(GridCachePartitionedNearDisabledMvccTxMultiThreadedSelfTest.class); + suite.addTestSuite(GridCachePartitionedMvccTxTimeoutSelfTest.class); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index 75b32fcf42ec9..a03a551e2e554 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.HashSet; import junit.framework.TestSuite; import org.apache.ignite.cache.affinity.rendezvous.ClusterNodeAttributeAffinityBackupFilterSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest; @@ -168,6 +169,7 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxTimeoutSelfTest; import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest; import org.apache.ignite.internal.processors.continuous.IgniteNoCustomEventsOnNodeStart; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -177,180 +179,197 @@ public class IgniteCacheTestSuite2 extends TestSuite { * @return IgniteCache test suite. * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + * @throws Exception Thrown in case of the failure. + */ + public static TestSuite suite(HashSet ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 2"); // Local cache. - suite.addTestSuite(GridCacheLocalBasicApiSelfTest.class); - suite.addTestSuite(GridCacheLocalBasicStoreSelfTest.class); - //suite.addTestSuite(GridCacheLocalBasicStoreMultithreadedSelfTest.class); - suite.addTestSuite(GridCacheLocalAtomicBasicStoreSelfTest.class); - suite.addTestSuite(GridCacheLocalGetAndTransformStoreSelfTest.class); - suite.addTestSuite(GridCacheLocalAtomicGetAndTransformStoreSelfTest.class); - suite.addTestSuite(GridCacheLocalLoadAllSelfTest.class); - suite.addTestSuite(GridCacheLocalLockSelfTest.class); - suite.addTestSuite(GridCacheLocalMultithreadedSelfTest.class); - suite.addTestSuite(GridCacheLocalTxSingleThreadedSelfTest.class); - //suite.addTestSuite(GridCacheLocalTxReadTest.class); - suite.addTestSuite(GridCacheLocalTxTimeoutSelfTest.class); - suite.addTestSuite(GridCacheLocalEventSelfTest.class); - suite.addTestSuite(GridCacheLocalEvictionEventSelfTest.class); - suite.addTestSuite(GridCacheVariableTopologySelfTest.class); - suite.addTestSuite(GridCacheLocalTxMultiThreadedSelfTest.class); - suite.addTestSuite(GridCacheTransformEventSelfTest.class); - suite.addTestSuite(GridCacheLocalIsolatedNodesSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalBasicApiSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalBasicStoreSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheLocalBasicStoreMultithreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalAtomicBasicStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalGetAndTransformStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalAtomicGetAndTransformStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalLoadAllSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalLockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalMultithreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalTxSingleThreadedSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheLocalTxReadTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalTxTimeoutSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalEvictionEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalTxMultiThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalIsolatedNodesSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, GridCacheTransformEventSelfTest.class, ignoredTests); // Partitioned cache. - suite.addTestSuite(GridCachePartitionedGetSelfTest.class); - suite.addTest(new TestSuite(GridCachePartitionedBasicApiTest.class)); - suite.addTest(new TestSuite(GridCacheNearMultiGetSelfTest.class)); - suite.addTest(new TestSuite(NoneRebalanceModeSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearJobExecutionSelfTest.class)); - suite.addTest(new TestSuite(GridCacheReplicatedJobExecutionTest.class)); - suite.addTest(new TestSuite(GridCacheNearOneNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCacheAtomicNearMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearReadersSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearReaderPreloadSelfTest.class)); - suite.addTest(new TestSuite(GridCacheAtomicNearReadersSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedAffinitySelfTest.class)); - //suite.addTest(new TestSuite(RendezvousAffinityFunctionSelfTest.class)); - suite.addTest(new TestSuite(RendezvousAffinityFunctionExcludeNeighborsSelfTest.class)); - suite.addTest(new TestSuite(RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.class)); - suite.addTest(new TestSuite(RendezvousAffinityFunctionStandardHashSelfTest.class)); - suite.addTest(new TestSuite(GridCacheRendezvousAffinityClientSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedProjectionAffinitySelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedBasicOpSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedBasicStoreSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedGetAndTransformStoreSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedAtomicGetAndTransformStoreSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedBasicStoreMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedNearDisabledBasicStoreMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedEventSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedLockSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedNearDisabledLockSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedMultiNodeLockSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedMultiThreadedPutGetSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedNodeFailureSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedExplicitLockNodeFailureSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedTxSingleThreadedSelfTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedEntryLockSelfTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedEvictionSelfTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedNestedTxTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedStorePutSelfTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedTxConcurrentGetTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedTxMultiNodeSelfTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedTxReadTest.class)); - suite.addTest(new TestSuite(GridCacheColocatedTxSingleThreadedSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedTxTimeoutSelfTest.class)); - suite.addTest(new TestSuite(GridCacheFinishPartitionsSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtEntrySelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtMappingSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedTxMultiThreadedSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedNearDisabledTxMultiThreadedSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadOnheapSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadBigDataSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadPutGetSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadDisabledSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadMultiThreadedSelfTest.class)); - suite.addTest(new TestSuite(CacheDhtLocalPartitionAfterRemoveSelfTest.class)); - suite.addTest(new TestSuite(GridCacheColocatedPreloadRestartSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearPreloadRestartSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadStartStopSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadUnloadSelfTest.class)); - suite.addTest(new TestSuite(RendezvousAffinityFunctionBackupFilterSelfTest.class)); - suite.addTest(new TestSuite(ClusterNodeAttributeAffinityBackupFilterSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedPreloadLifecycleSelfTest.class)); - suite.addTest(new TestSuite(CacheLoadingConcurrentGridStartSelfTest.class)); - suite.addTest(new TestSuite(CacheLoadingConcurrentGridStartSelfTestAllowOverwrite.class)); - suite.addTest(new TestSuite(CacheTxLoadingConcurrentGridStartSelfTestAllowOverwrite.class)); - suite.addTest(new TestSuite(GridCacheDhtPreloadDelayedSelfTest.class)); - suite.addTest(new TestSuite(GridPartitionedBackupLoadSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedLoadCacheSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionNotLoadedEventSelfTest.class)); - suite.addTest(new TestSuite(GridCacheDhtEvictionsDisabledSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearEvictionEventSelfTest.class)); - suite.addTest(new TestSuite(GridCacheAtomicNearEvictionEventSelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedGetSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedBasicApiTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedBasicOpSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearMultiGetSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, NoneRebalanceModeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearOneNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicNearMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearReadersSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearReaderPreloadSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicNearReadersSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedGetAndTransformStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedAtomicGetAndTransformStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedBasicStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridNearCacheStoreUpdateTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedStorePutSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedBasicStoreMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNearDisabledBasicStoreMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheConcurrentReadThroughTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedLockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNearDisabledLockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedMultiNodeLockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedMultiThreadedPutGetSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNodeFailureSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedExplicitLockNodeFailureSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheLockReleaseNodeLeaveTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedEntryLockSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedNestedTxTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedTxConcurrentGetTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedTxMultiNodeSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedTxReadTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxSingleThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheColocatedTxSingleThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxTimeoutSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheFinishPartitionsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTxMultiThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNearDisabledTxMultiThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtEntrySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtMappingSelfTest.class, ignoredTests); + + // Preload + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadOnheapSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadBigDataSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadPutGetSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadDisabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadMultiThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheColocatedPreloadRestartSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearPreloadRestartSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadStartStopSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadUnloadSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedPreloadLifecycleSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtPreloadDelayedSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, CacheDhtLocalPartitionAfterRemoveSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheLoadingConcurrentGridStartSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheLoadingConcurrentGridStartSelfTestAllowOverwrite.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheTxLoadingConcurrentGridStartSelfTestAllowOverwrite.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridPartitionedBackupLoadSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedLoadCacheSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionNotLoadedEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheDhtEvictionsDisabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearEvictionEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicNearEvictionEventSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedEvictionSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedTopologyChangeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedUnloadEventsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheColocatedOptimisticTransactionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicMessageCountSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearPartitionedClearSelfTest.class, ignoredTests); - suite.addTest(new TestSuite(GridCachePartitionedTopologyChangeSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedUnloadEventsSelfTest.class)); - suite.addTest(new TestSuite(GridCacheColocatedOptimisticTransactionSelfTest.class)); - suite.addTestSuite(GridCacheAtomicMessageCountSelfTest.class); - suite.addTest(new TestSuite(GridCacheNearPartitionedClearSelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, GridCacheOffheapUpdateSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearClientHitTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearPrimarySyncSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheColocatedPrimarySyncSelfTest.class, ignoredTests); - suite.addTest(new TestSuite(GridCacheOffheapUpdateSelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearClientHitTest.class)); - suite.addTest(new TestSuite(GridCacheNearPrimarySyncSelfTest.class)); - suite.addTest(new TestSuite(GridCacheColocatedPrimarySyncSelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionMapUpdateTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheClientNodePartitionsExchangeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheClientNodeChangingTopologyTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheServerNodeConcurrentStart.class, ignoredTests); - suite.addTest(new TestSuite(IgniteCachePartitionMapUpdateTest.class)); - suite.addTest(new TestSuite(IgniteCacheClientNodePartitionsExchangeTest.class)); - suite.addTest(new TestSuite(IgniteCacheClientNodeChangingTopologyTest.class)); - suite.addTest(new TestSuite(IgniteCacheServerNodeConcurrentStart.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryProcessorNodeJoinTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteAtomicCacheEntryProcessorNodeJoinTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearTxForceKeyTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CrossCacheTxRandomOperationsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CrossCacheTxNearEnabledRandomOperationsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteDynamicCacheAndNodeStop.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, NearCacheSyncUpdateTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheEnumOperationsSingleNodeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheEnumOperationsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheIncrementTxTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionedBackupNodeFailureRecoveryTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheVariableTopologySelfTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteCacheEntryProcessorNodeJoinTest.class)); - suite.addTest(new TestSuite(IgniteAtomicCacheEntryProcessorNodeJoinTest.class)); - suite.addTest(new TestSuite(GridCacheNearTxForceKeyTest.class)); - suite.addTest(new TestSuite(CrossCacheTxRandomOperationsTest.class)); - suite.addTest(new TestSuite(CrossCacheTxNearEnabledRandomOperationsTest.class)); - suite.addTest(new TestSuite(IgniteDynamicCacheAndNodeStop.class)); - suite.addTest(new TestSuite(CacheLockReleaseNodeLeaveTest.class)); - suite.addTest(new TestSuite(NearCacheSyncUpdateTest.class)); - suite.addTest(new TestSuite(CacheConfigurationLeakTest.class)); - suite.addTest(new TestSuite(MemoryPolicyConfigValidationTest.class)); - suite.addTest(new TestSuite(MemoryPolicyInitializationTest.class)); - suite.addTest(new TestSuite(CacheGroupLocalConfigurationSelfTest.class)); - suite.addTest(new TestSuite(CacheEnumOperationsSingleNodeTest.class)); - suite.addTest(new TestSuite(CacheEnumOperationsTest.class)); - suite.addTest(new TestSuite(IgniteCacheIncrementTxTest.class)); - suite.addTest(new TestSuite(IgniteCachePartitionedBackupNodeFailureRecoveryTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteNoCustomEventsOnNodeStart.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheExchangeMessageDuplicatedStateTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteNoCustomEventsOnNodeStart.class)); + //GridTestUtils.addTestIfNeeded(suite,NearCacheMultithreadedUpdateTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,NearCachePutAllMultinodeTest.class, ignoredTests); - suite.addTest(new TestSuite(CacheExchangeMessageDuplicatedStateTest.class)); - suite.addTest(new TestSuite(CacheConcurrentReadThroughTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteOnePhaseCommitInvokeTest.class, ignoredTests); - suite.addTest(new TestSuite(GridNearCacheStoreUpdateTest.class)); - //suite.addTest(new TestSuite(NearCacheMultithreadedUpdateTest.class)); - //suite.addTest(new TestSuite(NearCachePutAllMultinodeTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheNoSyncForGetTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheContainsKeyNearSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheNearTxRollbackTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteOnePhaseCommitInvokeTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteOnePhaseCommitNearReadersTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteNearClientCacheCloseTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClientCacheStartFailoverTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteCacheNoSyncForGetTest.class)); - //suite.addTest(new TestSuite(IgniteCacheContainsKeyNearSelfTest.class)); - //suite.addTest(new TestSuite(IgniteCacheNearTxRollbackTest.class)); + GridTestUtils.addTestIfNeeded(suite, CacheOptimisticTransactionsWithFilterSingleServerTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheOptimisticTransactionsWithFilterTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteOnePhaseCommitNearReadersTest.class)); - suite.addTest(new TestSuite(IgniteNearClientCacheCloseTest.class)); - suite.addTest(new TestSuite(IgniteClientCacheStartFailoverTest.class)); + GridTestUtils.addTestIfNeeded(suite, NonAffinityCoordinatorDynamicStartStopTest.class, ignoredTests); - suite.addTest(new TestSuite(CacheOptimisticTransactionsWithFilterSingleServerTest.class)); - suite.addTest(new TestSuite(CacheOptimisticTransactionsWithFilterTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheClearDuringRebalanceTest.class, ignoredTests); - suite.addTest(new TestSuite(NonAffinityCoordinatorDynamicStartStopTest.class)); + //GridTestUtils.addTestIfNeeded(suite,GridCacheColocatedDebugTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite, GridCacheDhtAtomicEvictionNearReadersSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheDhtEntrySetSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheDhtEvictionNearReadersSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheDhtMultiBackupTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheDhtPreloadMessageCountTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedNearDisabledMetricsSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheContainsKeyColocatedSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCrossCacheTxNearEnabledSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteTxConsistencyColocatedRestartSelfTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteCacheClearDuringRebalanceTest.class)); + // Configuration validation + GridTestUtils.addTestIfNeeded(suite, CacheConfigurationLeakTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, MemoryPolicyConfigValidationTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, MemoryPolicyInitializationTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheGroupLocalConfigurationSelfTest.class, ignoredTests); - suite.addTest(new TestSuite(CachePartitionStateTest.class)); + // Affinity and collocation + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedAffinitySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedProjectionAffinitySelfTest.class, ignoredTests); - suite.addTest(new TestSuite(CacheComparatorTest.class)); + // Other tests. + GridTestUtils.addTestIfNeeded(suite, GridCacheNearJobExecutionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheReplicatedJobExecutionTest.class, ignoredTests); - suite.addTest(new TestSuite(CachePartitionPartialCountersMapSelfTest.class)); + //GridTestUtils.addTestIfNeeded(suite,RendezvousAffinityFunctionSelfTest.class), ignoredTests); + GridTestUtils.addTestIfNeeded(suite, RendezvousAffinityFunctionExcludeNeighborsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, RendezvousAffinityFunctionStandardHashSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRendezvousAffinityClientSelfTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteReflectionFactorySelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, RendezvousAffinityFunctionBackupFilterSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, ClusterNodeAttributeAffinityBackupFilterSelfTest.class, ignoredTests); - //suite.addTest(new TestSuite(GridCacheColocatedDebugTest.class)); - //suite.addTest(new TestSuite(GridCacheDhtAtomicEvictionNearReadersSelfTest.class)); - //suite.addTest(new TestSuite(GridCacheDhtEntrySetSelfTest.class)); - //suite.addTest(new TestSuite(GridCacheDhtEvictionNearReadersSelfTest.class)); - //suite.addTest(new TestSuite(GridCacheDhtMultiBackupTest.class)); - //suite.addTest(new TestSuite(GridCacheDhtPreloadMessageCountTest.class)); - //suite.addTest(new TestSuite(GridCachePartitionedNearDisabledMetricsSelfTest.class)); - //suite.addTest(new TestSuite(IgniteCacheContainsKeyColocatedSelfTest.class)); - //suite.addTest(new TestSuite(IgniteCrossCacheTxNearEnabledSelfTest.class)); - //suite.addTest(new TestSuite(IgniteTxConsistencyColocatedRestartSelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, CachePartitionStateTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheComparatorTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CachePartitionPartialCountersMapSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteReflectionFactorySelfTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index eed89927a366e..293ac5e037f9c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -29,7 +29,6 @@ import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest; import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest; import org.apache.ignite.internal.processors.cache.distributed.CacheParallelStartTest; -import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest; import org.apache.ignite.internal.processors.cache.distributed.CacheTryLockMultithreadedTest; import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCache150ClientsTest; @@ -64,8 +63,6 @@ public class IgniteCacheTestSuite6 extends TestSuite { public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("IgniteCache Test Suite part 6"); - suite.addTestSuite(CachePartitionStateTest.class); - suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class); suite.addTestSuite(IgniteOptimisticTxSuspendResumeTest.class); suite.addTestSuite(IgniteOptimisticTxSuspendResumeMultiServerTest.class); From 66e0c328f414bdf6736a93accc46033891681ac7 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Tue, 27 Nov 2018 10:31:01 +0300 Subject: [PATCH 177/403] IGNITE-10393: MVCC: Fixed streamer with persistence on. Cherry-picked from f8307b24537146471d1dfec11f457f0852f15941 --- .../processors/cache/GridCacheMapEntry.java | 2 +- .../persistence/pagemem/PageMemoryImpl.java | 2 +- ...treamProcessorMvccPersistenceSelfTest.java | 28 +++++++++ ...ataStreamProcessorPersistenceSelfTest.java | 28 +++++++++ .../DataStreamProcessorSelfTest.java | 63 ++++++++++++++++++- .../IgniteBinaryCacheTestSuite.java | 2 + .../testsuites/IgniteCacheMvccTestSuite.java | 2 + .../testsuites/IgniteCacheTestSuite.java | 2 + 8 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccPersistenceSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorPersistenceSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 539c6fa3ae86a..341fd5c30e881 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3472,7 +3472,7 @@ else if (deletedUnlocked()) expireTime, partition(), updateCntr, - mvccVer + mvccVer == null ? MvccUtils.INITIAL_VERSION : mvccVer ))); } else { cctx.shared().wal().log(new DataRecord(new DataEntry( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 988b9364c0f15..c5cdfcee5bb4b 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -53,7 +53,6 @@ import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; -import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; @@ -65,6 +64,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker; import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier; import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; +import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO; import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccPersistenceSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccPersistenceSelfTest.java new file mode 100644 index 0000000000000..9360cab60284e --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccPersistenceSelfTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.datastreamer; + +/** + * + */ +public class DataStreamProcessorMvccPersistenceSelfTest extends DataStreamProcessorMvccSelfTest { + /** {@inheritDoc} */ + @Override public boolean persistenceEnabled() { + return true; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorPersistenceSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorPersistenceSelfTest.java new file mode 100644 index 0000000000000..7ce4fdd28e9d3 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorPersistenceSelfTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.datastreamer; + +/** + * + */ +public class DataStreamProcessorPersistenceSelfTest extends DataStreamProcessorSelfTest { + /** {@inheritDoc} */ + @Override public boolean persistenceEnabled() { + return true; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java index 536d73e271d59..e7f43a495d706 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java @@ -42,9 +42,12 @@ import org.apache.ignite.cache.store.CacheStoreAdapter; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.IgniteReflectionFactory; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.configuration.WALMode; import org.apache.ignite.events.Event; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; @@ -52,6 +55,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter; import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteClosure; @@ -97,6 +101,13 @@ public class DataStreamProcessorSelfTest extends GridCommonAbstractTest { /** */ private TestStore store; + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + if (persistenceEnabled()) + cleanPersistenceDir(); + } + /** {@inheritDoc} */ @Override public void afterTest() throws Exception { super.afterTest(); @@ -104,6 +115,13 @@ public class DataStreamProcessorSelfTest extends GridCommonAbstractTest { useCache = false; } + /** + * @return {@code True} if persistent store is enabled for test. + */ + public boolean persistenceEnabled() { + return false; + } + /** {@inheritDoc} */ @SuppressWarnings({"IfMayBeConditional", "unchecked"}) @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -141,6 +159,12 @@ public class DataStreamProcessorSelfTest extends GridCommonAbstractTest { } cfg.setCacheConfiguration(cc); + + if (persistenceEnabled()) + cfg.setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true)) + .setWalMode(WALMode.LOG_ONLY)); } else { cfg.setCacheConfiguration(); @@ -226,6 +250,8 @@ private void checkDataStreamer() throws Exception { Ignite igniteWithoutCache = startGrid(1); + afterGridStarted(); + final IgniteDataStreamer ldr = igniteWithoutCache.dataStreamer(DEFAULT_CACHE_NAME); ldr.receiver(DataStreamerCacheUpdaters.batchedSorted()); @@ -338,7 +364,7 @@ private void checkIsolatedDataStreamer() throws Exception { startGrid(1); startGrid(2); - awaitPartitionMapExchange(); + afterGridStarted(); IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); @@ -423,6 +449,8 @@ public void testPrimitiveArrays() throws Exception { Ignite g1 = startGrid(1); startGrid(2); // Reproduced only for several nodes in topology (if marshalling is used). + afterGridStarted(); + List arrays = Arrays.asList( new byte[] {1}, new boolean[] {true, false}, new char[] {2, 3}, new short[] {3, 4}, new int[] {4, 5}, new long[] {5, 6}, new float[] {6, 7}, new double[] {7, 8}); @@ -486,6 +514,8 @@ protected void checkLoaderMultithreaded(int nodesCntNoCache, int nodesCntCache) Ignite g1 = grid(idx - 1); + afterGridStarted(); + // Get and configure loader. final IgniteDataStreamer ldr = g1.dataStreamer(DEFAULT_CACHE_NAME); @@ -590,6 +620,8 @@ public void testLoaderApi() throws Exception { try { Ignite g1 = startGrid(1); + afterGridStarted(); + IgniteDataStreamer ldr = g1.dataStreamer(DEFAULT_CACHE_NAME); ldr.close(false); @@ -747,6 +779,8 @@ public void testFlush() throws Exception { try { Ignite g = startGrid(); + afterGridStarted(); + final IgniteCache c = g.cache(DEFAULT_CACHE_NAME); final IgniteDataStreamer ldr = g.dataStreamer(DEFAULT_CACHE_NAME); @@ -800,6 +834,8 @@ public void testTryFlush() throws Exception { try { Ignite g = startGrid(); + afterGridStarted(); + IgniteCache c = g.cache(DEFAULT_CACHE_NAME); IgniteDataStreamer ldr = g.dataStreamer(DEFAULT_CACHE_NAME); @@ -836,6 +872,8 @@ public void testFlushTimeout() throws Exception { try { Ignite g = startGrid(); + afterGridStarted(); + final CountDownLatch latch = new CountDownLatch(9); g.events().localListen(new IgnitePredicate() { @@ -892,6 +930,8 @@ public void testUpdateStore() throws Exception { startGrid(2); startGrid(3); + afterGridStarted(); + for (int i = 0; i < 1000; i++) storeMap.put(i, i); @@ -941,6 +981,8 @@ public void testUpdateStore() throws Exception { } finally { storeMap = null; + + stopAllGrids(); } } @@ -956,6 +998,8 @@ public void testCustomUserUpdater() throws Exception { startGrid(2); startGrid(3); + afterGridStarted(); + try (IgniteDataStreamer ldr = ignite.dataStreamer(DEFAULT_CACHE_NAME)) { ldr.allowOverwrite(true); ldr.keepBinary(customKeepBinary()); @@ -989,6 +1033,8 @@ public void testLocalDataStreamerDedicatedThreadPool() throws Exception { Ignite ignite = startGrid(1); + afterGridStarted(); + final IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); try (IgniteDataStreamer ldr = ignite.dataStreamer(DEFAULT_CACHE_NAME)) { @@ -1035,6 +1081,8 @@ public void testRemoteDataStreamerDedicatedThreadPool() throws Exception { Ignite client = startGrid(0); + afterGridStarted(); + final IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); try (IgniteDataStreamer ldr = client.dataStreamer(DEFAULT_CACHE_NAME)) { @@ -1100,6 +1148,19 @@ protected StreamReceiver getStreamReceiver() { return new TestDataReceiver(); } + /** + * Activates grid if necessary and wait for partition map exchange. + */ + private void afterGridStarted() throws InterruptedException { + G.allGrids().stream() + .filter(g -> !g.cluster().node().isClient()) + .findAny() + .filter(g -> !g.cluster().active()) + .ifPresent(g -> g.cluster().active(true)); + + awaitPartitionMapExchange(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java index 170bb33c9ba41..5cfe534d4c3b6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryMultithreadedSelfTest; import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheBinariesNearPartitionedByteArrayValuesSelfTest; import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheBinariesPartitionedOnlyByteArrayValuesSelfTest; +import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorPersistenceSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorSelfTest; /** @@ -51,6 +52,7 @@ public static TestSuite suite() throws Exception { // Tests below have a special version for Binary Marshaller ignoredTests.add(DataStreamProcessorSelfTest.class); + ignoredTests.add(DataStreamProcessorPersistenceSelfTest.class); ignoredTests.add(GridCacheAffinityRoutingSelfTest.class); ignoredTests.add(IgniteCacheAtomicLocalExpiryPolicyTest.class); ignoredTests.add(GridCacheEntryMemorySizeSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java index d4b837cf9dbe2..930706d497b45 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java @@ -38,6 +38,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccVacuumTest; import org.apache.ignite.internal.processors.cache.mvcc.MvccCachePeekTest; import org.apache.ignite.internal.processors.cache.mvcc.MvccUnsupportedTxModesTest; +import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccPersistenceSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; @@ -59,6 +60,7 @@ public static TestSuite suite() { suite.addTestSuite(CacheMvccConfigurationValidationTest.class); suite.addTestSuite(DataStreamProcessorMvccSelfTest.class); + suite.addTestSuite(DataStreamProcessorMvccPersistenceSelfTest.class); suite.addTestSuite(CacheMvccOperationChecksTest.class); suite.addTestSuite(CacheMvccRemoteTxOnNearNodeStartTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java index 905428482b4f4..eeec5e8b52482 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java @@ -169,6 +169,7 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxExceptionSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheEntryProcessorExternalizableFailedTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheEntryProcessorNonSerializableTest; +import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorPersistenceSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamerClientReconnectAfterClusterRestartTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamerImplSelfTest; @@ -275,6 +276,7 @@ public static TestSuite suite(Set ignoredTests) throws Exception { suite.addTestSuite(GridCacheBalancingStoreSelfTest.class); suite.addTestSuite(GridCacheAffinityApiSelfTest.class); suite.addTestSuite(GridCacheStoreValueBytesSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorPersistenceSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DataStreamerUpdateAfterLoadTest.class, ignoredTests); suite.addTestSuite(DataStreamerMultiThreadedSelfTest.class); From 5c6fa006a2648f3473f09cd4d067a2f32f1c1871 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Tue, 27 Nov 2018 13:42:45 +0300 Subject: [PATCH 178/403] IGNITE-10393: MVCC: Fixed streamer with persistence on. Cherry-picked from 038bb75d46cce8b950641e2715a22b9b693a6339 --- .../org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java index 930706d497b45..6aede3b540f14 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java @@ -36,8 +36,6 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTransactionsTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTxFailoverTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccVacuumTest; -import org.apache.ignite.internal.processors.cache.mvcc.MvccCachePeekTest; -import org.apache.ignite.internal.processors.cache.mvcc.MvccUnsupportedTxModesTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccPersistenceSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest; From ec0c337c7c19c94ff523374e25b4bfeae5fe3e4b Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Tue, 27 Nov 2018 13:56:02 +0300 Subject: [PATCH 179/403] Fix compilation by partial backport of IGNITE-10002 --- .../cache/IgniteCacheEntryProcessorNodeJoinTest.java | 3 ++- .../distributed/GridCachePartitionNotLoadedEventSelfTest.java | 1 + .../org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java index 89d771ee43dd6..5ac62a11ba5e1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorNodeJoinTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -427,4 +428,4 @@ private Processor(String val) { return vals.size(); } } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java index 801495f610cd9..94ef32c494996 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionNotLoadedEventSelfTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.util.TestTcpCommunicationSpi; import org.eclipse.jetty.util.ConcurrentHashSet; diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java index 6aede3b540f14..930706d497b45 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java @@ -36,6 +36,8 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTransactionsTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTxFailoverTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccVacuumTest; +import org.apache.ignite.internal.processors.cache.mvcc.MvccCachePeekTest; +import org.apache.ignite.internal.processors.cache.mvcc.MvccUnsupportedTxModesTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccPersistenceSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest; From 74452149bc56eea8a402f3f9a75ceb50a8ea1e48 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Tue, 27 Nov 2018 15:08:59 +0300 Subject: [PATCH 180/403] IGNITE-10002: MVCC: Create "Cache 2" test suite for MVCC mode. This closes #5198. fixed imports --- .../cache/CacheConcurrentReadThroughTest.java | 1 - .../IgniteClientCacheStartFailoverTest.java | 2 +- .../IgniteCacheTxIteratorSelfTest.java | 6 +---- .../testsuites/IgniteCacheTestSuite2.java | 24 ------------------- .../testsuites/IgniteCacheTestSuite6.java | 1 - 5 files changed, 2 insertions(+), 32 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java index b2842e09730ae..1e8ed4ddf2be4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConcurrentReadThroughTest.java @@ -24,7 +24,6 @@ import javax.cache.configuration.Factory; import javax.cache.integration.CacheLoaderException; import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCompute; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.store.CacheStoreAdapter; import org.apache.ignite.configuration.CacheConfiguration; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java index fa7d0a879ec73..c8025528764dc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java @@ -41,8 +41,8 @@ import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiPredicate; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java index acb800ac54d7a..d6fb35dcc5e77 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.distributed; +import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; @@ -27,9 +28,6 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; -import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; -import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.MvccFeatureChecker.Feature; @@ -38,8 +36,6 @@ import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; -import javax.cache.Cache; - /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index a03a551e2e554..179f5e95cff35 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -23,7 +23,6 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionExcludeNeighborsSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionFastPowerOfTwoHashSelfTest; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionStandardHashSelfTest; import org.apache.ignite.internal.IgniteReflectionFactorySelfTest; import org.apache.ignite.internal.processors.cache.CacheComparatorTest; @@ -68,22 +67,16 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheClientNodePartitionsExchangeTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheServerNodeConcurrentStart; import org.apache.ignite.internal.processors.cache.distributed.dht.CachePartitionPartialCountersMapSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedDebugTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedOptimisticTransactionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPreloadRestartSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPrimarySyncSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedTxSingleThreadedSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtAtomicEvictionNearReadersSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtEntrySelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtEntrySetSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtEvictionNearReadersSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtEvictionsDisabledSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtMappingSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtMultiBackupTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadBigDataSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadDelayedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadDisabledSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadMessageCountTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadMultiThreadedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadOnheapSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadPutGetSelfTest; @@ -91,14 +84,10 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadStartStopSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadUnloadSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledLockSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledMetricsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedTopologyChangeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedUnloadEventsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheClearDuringRebalanceTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheContainsKeyColocatedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePartitionedBackupNodeFailureRecoveryTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCrossCacheTxNearEnabledSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteTxConsistencyColocatedRestartSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearEvictionEventSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearMultiNodeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearReadersSelfTest; @@ -120,9 +109,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedBasicOpSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedBasicStoreMultiNodeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedBasicStoreSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedEntryLockSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedEventSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedEvictionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedExplicitLockNodeFailureSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedGetAndTransformStoreSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedLoadCacheSelfTest; @@ -131,30 +118,20 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiThreadedPutGetSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNearDisabledBasicStoreMultiNodeSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNestedTxTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNodeFailureSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedPreloadLifecycleSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedStorePutSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxConcurrentGetTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxMultiNodeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxMultiThreadedSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxReadTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxSingleThreadedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxTimeoutSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheRendezvousAffinityClientSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheStoreUpdateTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridPartitionedBackupLoadSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheContainsKeyNearSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheNearTxRollbackTest; -import org.apache.ignite.internal.processors.cache.distributed.near.NearCacheMultithreadedUpdateTest; -import org.apache.ignite.internal.processors.cache.distributed.near.NearCachePutAllMultinodeTest; import org.apache.ignite.internal.processors.cache.distributed.near.NearCacheSyncUpdateTest; import org.apache.ignite.internal.processors.cache.distributed.near.NoneRebalanceModeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedJobExecutionTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicBasicStoreSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicGetAndTransformStoreSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalBasicApiSelfTest; -import org.apache.ignite.internal.processors.cache.local.GridCacheLocalBasicStoreMultithreadedSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalBasicStoreSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalEventSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalEvictionEventSelfTest; @@ -164,7 +141,6 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheLocalLockSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalMultithreadedSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxMultiThreadedSelfTest; -import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxReadTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxSingleThreadedSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxTimeoutSelfTest; import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest; diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index 293ac5e037f9c..7bb476f0b70f7 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeMultiServerTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest; import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest; -import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingOrderingTest; import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest; import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest; import org.apache.ignite.internal.processors.cache.transactions.TxOnCachesStartTest; From 415bf33af6d822d51467bf27017643b2e54eba02 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Tue, 27 Nov 2018 19:26:26 +0300 Subject: [PATCH 181/403] IGNITE-7072: IgniteCache.replace(k, v, nv) fix for binary mode (cherry picked from commit 23de74357483aa2979b54ea4e54254ef0d2f3637) --- .../CacheEntryPredicateContainsValue.java | 4 ++ ...ridCacheBinaryObjectsAbstractSelfTest.java | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java index 73a98b74ac0cf..ad9861cf8b24e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -60,6 +61,9 @@ public CacheEntryPredicateContainsValue(CacheObject val) { GridCacheContext cctx = e.context(); + if (this.val instanceof BinaryObject && val instanceof BinaryObject) + return F.eq(val, this.val); + Object thisVal = CU.value(this.val, cctx, false); Object cacheVal = CU.value(val, cctx, false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java index cc2f2c4cb2011..3a2fe199d0c27 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java @@ -287,6 +287,64 @@ public void testGet() throws Exception { } } + /** + * @throws Exception If failed. + */ + public void testReplace() throws Exception { + IgniteCache c = jcache(0); + + for (int i = 0; i < ENTRY_CNT; i++) + c.put(i, new TestObject(i)); + + for (int i = 0; i < ENTRY_CNT; i++) { + TestObject obj = c.get(i); + + assertEquals(i, obj.val); + } + + IgniteCache kpc = keepBinaryCache(); + + BinaryObjectBuilder bldr = grid(0).binary().builder(TestObject.class.getName()); + + bldr.setField("val", -42); + + BinaryObject testObj = bldr.build(); + + for (int i = 0; i < ENTRY_CNT; i++) { + BinaryObject po = kpc.get(i); + + assertEquals(i, (int)po.field("val")); + + assertTrue(kpc.replace(i, po, testObj)); + } + } + + /** + * @throws Exception If failed. + */ + public void testRemove() throws Exception { + IgniteCache c = jcache(0); + + for (int i = 0; i < ENTRY_CNT; i++) + c.put(i, new TestObject(i)); + + for (int i = 0; i < ENTRY_CNT; i++) { + TestObject obj = c.get(i); + + assertEquals(i, obj.val); + } + + IgniteCache kpc = keepBinaryCache(); + + for (int i = 0; i < ENTRY_CNT; i++) { + BinaryObject po = kpc.get(i); + + assertEquals(i, (int)po.field("val")); + + assertTrue(kpc.remove(i, po)); + } + } + /** * @throws Exception If failed. */ From 1951500e652754bd484720bd78144484d40a77c2 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Fri, 16 Nov 2018 19:11:16 +0300 Subject: [PATCH 182/403] IGNITE-10156 Fixed invalid conversion of DynamicCacheDescriptor to StoredCacheData - Fixes #5332. Signed-off-by: Alexey Goncharuk (cherry picked from commit 2311afd) --- .../apache/ignite/internal/processors/cache/CachesRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java index fe55f979a56ec..d37f69ca5f6ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java @@ -280,7 +280,7 @@ private boolean shouldPersist(CacheConfiguration cacheCfg) { */ private IgniteInternalFuture persistCacheConfigurations(List cacheDescriptors) { List cacheConfigsToPersist = cacheDescriptors.stream() - .map(cacheDesc -> new StoredCacheData(cacheDesc.cacheConfiguration()).sql(cacheDesc.sql())) + .map(DynamicCacheDescriptor::toStoredData) .collect(Collectors.toList()); // Pre-create cache work directories if they don't exist. From ac37c4e1161c0e6c37792de02d74b8e70229ca36 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 28 Nov 2018 11:11:20 +0300 Subject: [PATCH 183/403] IGNITE-10432: Fixed data types in world.sql. --- examples/sql/world.sql | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/sql/world.sql b/examples/sql/world.sql index a34ee71fbf5dd..829fba12abeaf 100644 --- a/examples/sql/world.sql +++ b/examples/sql/world.sql @@ -2,30 +2,30 @@ DROP TABLE IF EXISTS Country; CREATE TABLE Country ( Code CHAR(3) PRIMARY KEY, - Name CHAR(52), - Continent CHAR(50), - Region CHAR(26), + Name VARCHAR, + Continent VARCHAR, + Region VARCHAR, SurfaceArea DECIMAL(10,2), - IndepYear SMALLINT(6), - Population INT(11), + IndepYear SMALLINT, + Population INT, LifeExpectancy DECIMAL(3,1), GNP DECIMAL(10,2), GNPOld DECIMAL(10,2), - LocalName CHAR(45), - GovernmentForm CHAR(45), - HeadOfState CHAR(60), - Capital INT(11), + LocalName VARCHAR, + GovernmentForm VARCHAR, + HeadOfState VARCHAR, + Capital INT, Code2 CHAR(2) ) WITH "template=partitioned, backups=1, CACHE_NAME=Country, VALUE_TYPE=demo.model.Country"; DROP TABLE IF EXISTS City; CREATE TABLE City ( - ID INT(11), - Name CHAR(35), + ID INT, + Name VARCHAR, CountryCode CHAR(3), - District CHAR(20), - Population INT(11), + District VARCHAR, + Population INT, PRIMARY KEY (ID, CountryCode) ) WITH "template=partitioned, backups=1, affinityKey=CountryCode, CACHE_NAME=City, KEY_TYPE=demo.model.CityKey, VALUE_TYPE=demo.model.City"; @@ -35,7 +35,7 @@ DROP TABLE IF EXISTS CountryLanguage; CREATE TABLE CountryLanguage ( CountryCode CHAR(3), - Language CHAR(30), + Language VARCHAR, IsOfficial CHAR(2), Percentage DECIMAL(4,1), PRIMARY KEY (CountryCode, Language) From d826dcf4c1e9120b5b3f83d5b55eb120ee7e45e6 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 28 Nov 2018 13:32:50 +0300 Subject: [PATCH 184/403] IGNITE-10298 Cover possible deadlock in case of caches start and frequent checkpoints. - Fixes #5517. Signed-off-by: Pavel Kovalenko (cherry picked from commit fe8c8cc) --- .../GridCacheDatabaseSharedManager.java | 20 +- ...sCacheStartStopWithFreqCheckpointTest.java | 177 ++++++++++++++++++ .../testsuites/IgnitePdsTestSuite2.java | 3 + 3 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index a2afe02fda967..d1dee32da9aac 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -249,6 +249,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */ private volatile Checkpointer checkpointer; + /** Checkpointer thread instance. */ + private volatile IgniteThread checkpointerThread; + /** For testing only. */ private volatile boolean checkpointsEnabled = true; @@ -417,6 +420,15 @@ public Checkpointer getCheckpointer() { return checkpointer; } + /** + * For test use only. + * + * @return Checkpointer thread instance. + */ + public IgniteThread checkpointerThread() { + return checkpointerThread; + } + /** * For test use only. */ @@ -1961,7 +1973,11 @@ private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { @Override public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException { long time = System.currentTimeMillis(); - new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start(); + IgniteThread cpThread = new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer); + + cpThread.start(); + + checkpointerThread = cpThread; CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started"); @@ -4595,7 +4611,7 @@ private static void dumpPartitionsInfo(CacheGroupContext grp, IgniteLogger log) pageStore.ensure(grp.groupId(), p); if (pageStore.pages(grp.groupId(), p) <= 1) { - log.info("Partition [id=" + p + ", state=N/A (only file header) ]"); + log.info("Partition [grp=" + grp.cacheOrGroupName() + ", id=" + p + ", state=N/A (only file header) ]"); continue; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java new file mode 100644 index 0000000000000..8e00d8871dc36 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheRebalanceMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Assert; + +/** + * + */ +public class IgnitePdsCacheStartStopWithFreqCheckpointTest extends GridCommonAbstractTest { + /** Caches. */ + private static final int CACHES = 10; + + /** Cache name. */ + private static final String CACHE_NAME = "test"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setConsistentId(igniteInstanceName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setWalMode(WALMode.LOG_ONLY) + .setCheckpointFrequency(1000) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setMaxSize(512 * 1024 * 1024) + .setPersistenceEnabled(true) + ); + + cfg.setDataStorageConfiguration(dsCfg); + + CacheConfiguration[] ccfgs = new CacheConfiguration[CACHES]; + + for (int i = 0; i < ccfgs.length; i++) + ccfgs[i] = cacheConfiguration(i); + + cfg.setCacheConfiguration(ccfgs); + + return cfg; + } + + /** {@inheritDoc} */ + private CacheConfiguration cacheConfiguration(int cacheIdx) { + return new CacheConfiguration(CACHE_NAME + cacheIdx) + .setCacheMode(CacheMode.REPLICATED) + .setBackups(0) + .setRebalanceMode(CacheRebalanceMode.NONE); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** + * Test checkpoint deadlock during caches start/stop process and frequent checkpoints is set. + * + * @throws Exception If failed. + */ + public void testCheckpointDeadlock() throws Exception { + IgniteEx crd = startGrid(0); + + crd.cluster().active(true); + + for (int cacheId = 0; cacheId < CACHES; cacheId++) { + IgniteCache cache = crd.getOrCreateCache(CACHE_NAME + cacheId); + + for (int key = 0; key < 4096; key++) + cache.put(key, key); + } + + forceCheckpoint(); + + final AtomicBoolean stopFlag = new AtomicBoolean(); + + IgniteInternalFuture cacheStartStopFut = GridTestUtils.runAsync(() -> { + while (!stopFlag.get()) { + List cacheNames = new ArrayList<>(); + for (int i = 0; i < CACHES / 2; i++) + cacheNames.add(CACHE_NAME + i); + + try { + // Stop cache without destroy. + crd.context().cache().dynamicDestroyCaches(cacheNames, false, false, false).get(); + } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to destroy cache", e); + } + + List cachesToStart = new ArrayList<>(); + for (int i = 0; i < CACHES / 2; i++) + cachesToStart.add(cacheConfiguration(i)); + + crd.getOrCreateCaches(cachesToStart); + } + }); + + U.sleep(60_000); + + log.info("Stopping caches start/stop process."); + + stopFlag.set(true); + + try { + cacheStartStopFut.get(30, TimeUnit.SECONDS); + } + catch (IgniteFutureTimeoutCheckedException e) { + U.dumpThreads(log); + + log.warning("Caches start/stop future hangs. Interrupting checkpointer..."); + + interruptCheckpointer(crd); + + // Should succeed. + cacheStartStopFut.get(); + + Assert.assertTrue("Checkpoint and exchange is probably in deadlock (see thread dump above for details).", false); + } + } + + /** + * Interrupts checkpoint thread for given node. + * + * @param node Node. + */ + private void interruptCheckpointer(IgniteEx node) { + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) node.context().cache().context().database(); + + dbMgr.checkpointerThread().interrupt(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index c12f515d461b6..baaf88254f4f2 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -19,6 +19,7 @@ import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.persistence.IgniteDataStorageMetricsSelfTest; +import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheStartStopWithFreqCheckpointTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest; @@ -117,6 +118,8 @@ private static void addRealPageStoreTestsNotForDirectIo(TestSuite suite) { suite.addTestSuite(IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.class); suite.addTestSuite(IgniteWalFlushFsyncWithMmapBufferSelfTest.class); + + suite.addTestSuite(IgnitePdsCacheStartStopWithFreqCheckpointTest.class); } /** From 6fa222aa6b679fbd06eda6b56dec3bf2972ed06f Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Wed, 28 Nov 2018 14:52:14 +0700 Subject: [PATCH 185/403] IGNITE-10433 Web Console: Fixed "Import models" dialog cleanup on exit logic. (cherry picked from commit 3606b21bfe0d254864b284571a3f2692d4d4870a) --- .../components/modal-import-models/service.js | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js index 3c0ecbbadc8b4..e17721dbc2b33 100644 --- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js +++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js @@ -16,12 +16,18 @@ */ export default class ModalImportModels { - static $inject = ['$modal', 'AgentManager', '$uiRouter']; - constructor($modal, AgentManager, $uiRouter) { - Object.assign(this, {$modal, AgentManager, $uiRouter}); + static $inject = ['$modal', '$uiRouter', 'AgentManager']; + + constructor($modal, $uiRouter, AgentManager) { + this.$modal = $modal; + this.$uiRouter = $uiRouter; + this.AgentManager = AgentManager; } + _goToDynamicState() { - if (this._state) this.$uiRouter.stateRegistry.deregister(this._state); + if (this._state) + this.$uiRouter.stateRegistry.deregister(this._state); + this._state = this.$uiRouter.stateRegistry.register({ name: 'importModels', parent: this.$uiRouter.stateService.current, @@ -29,11 +35,15 @@ export default class ModalImportModels { this._open(); }, onExit: () => { + this.AgentManager.stopWatch(); + this._modal && this._modal.hide(); } }); + return this.$uiRouter.stateService.go(this._state, this.$uiRouter.stateService.params); } + _open() { this._modal = this.$modal({ template: ` @@ -46,10 +56,12 @@ export default class ModalImportModels { controllerAs: '$ctrl', show: false }); + return this.AgentManager.startAgentWatch('Back', this.$uiRouter.globals.current.name) - .then(() => this._modal.$promise) - .then(() => this._modal.show()); + .then(() => this._modal.$promise) + .then(() => this._modal.show()); } + open() { this._goToDynamicState(); } From 85316f68e2368c2c0b82373bc72c39f7ec694319 Mon Sep 17 00:00:00 2001 From: "d.garus" Date: Thu, 29 Nov 2018 11:25:59 +0300 Subject: [PATCH 186/403] IGNITE-8640 Fixed dynamic cache creation failures - Fixes #4480. Signed-off-by: Alexey Goncharuk --- .../cache/CacheAffinitySharedManager.java | 16 +- ...niteAbstractDynamicCacheStartFailTest.java | 1959 ++++++++++------- .../CacheMvccConfigurationValidationTest.java | 37 +- .../query/SqlIllegalSchemaSelfTest.java | 176 +- .../FailureHandlerTest.cs | 2 + 5 files changed, 1354 insertions(+), 836 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 881f8dbb1e5b8..ce87fa94776e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -924,14 +924,18 @@ private void processCacheStartRequests( Map failedCaches = cctx.cache().prepareStartCachesIfPossible(startCacheInfos.keySet()); - failedCaches.forEach((cacheInfo, exception) -> { - U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " + - "[cacheName=" + cacheInfo.getStartedConfiguration().getName() + ']', exception); + for (Map.Entry entry : failedCaches.entrySet()) { + if (cctx.localNode().isClient()) { + U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " + + "[cacheName=" + entry.getKey().getStartedConfiguration().getName() + ']', entry.getValue()); - cctx.cache().closeCaches(Collections.singleton(cacheInfo.getStartedConfiguration().getName()), false); + cctx.cache().closeCaches(Collections.singleton(entry.getKey().getStartedConfiguration().getName()), false); - cctx.cache().completeCacheStartFuture(startCacheInfos.get(cacheInfo), false, exception); - }); + cctx.cache().completeCacheStartFuture(startCacheInfos.get(entry.getKey()), false, entry.getValue()); + } + else + throw entry.getValue(); + } Set failedCacheInfos = failedCaches.keySet(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java index db13c11d1a947..d506224dc1901 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java @@ -1,786 +1,1173 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; -import javax.cache.CacheException; -import javax.cache.configuration.Factory; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteDataStreamer; -import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.affinity.AffinityFunctionContext; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.cache.query.annotations.QuerySqlField; -import org.apache.ignite.cache.store.CacheStore; -import org.apache.ignite.cluster.BaselineNode; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.lang.IgnitePredicate; -import org.apache.ignite.resources.IgniteInstanceResource; -import org.apache.ignite.testframework.GridTestUtils; - -/** - * Tests the recovery after a dynamic cache start failure. - */ -public abstract class IgniteAbstractDynamicCacheStartFailTest extends GridCacheAbstractSelfTest { - /** */ - private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache"; - - /** */ - private static final String CLIENT_GRID_NAME = "client"; - - /** */ - protected static final String EXISTING_CACHE_NAME = "existing-cache";; - - /** */ - private static final int PARTITION_COUNT = 16; - - /** Coordinator node index. */ - private int crdIdx = 0; - - /** {@inheritDoc} */ - @Override protected int gridCount() { - return 3; - } - - /** - * Returns {@code true} if persistence is enabled. - * - * @return {@code true} if persistence is enabled. - */ - protected boolean persistenceEnabled() { - return false; - } - - /** - * @throws Exception If failed. - */ - public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception { - final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient"; - - IgniteConfiguration clientCfg = getConfiguration(clientName); - - clientCfg.setClientMode(true); - - Ignite client = startGrid(clientName, clientCfg); - - CacheConfiguration cfg = new CacheConfiguration(); - - cfg.setName(DYNAMIC_CACHE_NAME + "-server-1"); - - cfg.setAffinity(new BrokenAffinityFunction(false, clientName)); - - try { - IgniteCache cache = ignite(0).getOrCreateCache(cfg); - } - catch (CacheException e) { - fail("Exception should not be thrown."); - } - - stopGrid(clientName); - } - - /** - * @throws Exception If failed. - */ - public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception { - final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer"; - - IgniteConfiguration clientCfg = getConfiguration(clientName); - - clientCfg.setClientMode(true); - - Ignite client = startGrid(clientName, clientCfg); - - CacheConfiguration cfg = new CacheConfiguration(); - - cfg.setName(DYNAMIC_CACHE_NAME + "-server-2"); - - cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0))); - - try { - IgniteCache cache = ignite(0).getOrCreateCache(cfg); - - fail("Expected exception was not thrown."); - } - catch (CacheException e) { - } - - stopGrid(clientName); - } - - /** - * @throws Exception If failed. - */ - public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception { - final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer"; - - IgniteConfiguration clientCfg = getConfiguration(clientName); - - clientCfg.setClientMode(true); - - Ignite client = startGrid(clientName, clientCfg); - - CacheConfiguration cfg = new CacheConfiguration(); - - cfg.setName(DYNAMIC_CACHE_NAME + "-client-2"); - - cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0))); - - try { - IgniteCache cache = client.getOrCreateCache(cfg); - - fail("Expected exception was not thrown."); - } - catch (CacheException e) { - } - - stopGrid(clientName); - } - - /** - * Test cache start with broken affinity function that throws an exception on all nodes. - */ - public void testBrokenAffinityFunOnAllNodes() { - final boolean failOnAllNodes = true; - final int unluckyNode = 0; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = 0; - - testDynamicCacheStart( - createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Test cache start with broken affinity function that throws an exception on initiator node. - */ - public void testBrokenAffinityFunOnInitiator() { - final boolean failOnAllNodes = false; - final int unluckyNode = 1; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = 1; - - testDynamicCacheStart( - createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Test cache start with broken affinity function that throws an exception on non-initiator node. - */ - public void testBrokenAffinityFunOnNonInitiator() { - final boolean failOnAllNodes = false; - final int unluckyNode = 1; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = 2; - - testDynamicCacheStart( - createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Test cache start with broken affinity function that throws an exception on coordinator node. - */ - public void testBrokenAffinityFunOnCoordinatorDiffInitiator() { - final boolean failOnAllNodes = false; - final int unluckyNode = crdIdx; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = (crdIdx + 1) % gridCount(); - - testDynamicCacheStart( - createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Test cache start with broken affinity function that throws an exception on initiator node. - */ - public void testBrokenAffinityFunOnCoordinator() { - final boolean failOnAllNodes = false; - final int unluckyNode = crdIdx; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = crdIdx; - - testDynamicCacheStart( - createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Tests cache start with node filter and broken affinity function that throws an exception on initiator node. - */ - public void testBrokenAffinityFunWithNodeFilter() { - final boolean failOnAllNodes = false; - final int unluckyNode = 0; - final int unluckyCfg = 0; - final int numOfCaches = 1; - final int initiator = 0; - - testDynamicCacheStart( - createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true), - initiator); - } - - /** - * Tests cache start with broken cache store that throws an exception on all nodes. - */ - public void testBrokenCacheStoreOnAllNodes() { - final boolean failOnAllNodes = true; - final int unluckyNode = 0; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = 0; - - testDynamicCacheStart( - createCacheConfigsWithBrokenCacheStore( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Tests cache start with broken cache store that throws an exception on initiator node. - */ - public void testBrokenCacheStoreOnInitiator() { - final boolean failOnAllNodes = false; - final int unluckyNode = 1; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = 1; - - testDynamicCacheStart( - createCacheConfigsWithBrokenCacheStore( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Tests cache start with broken cache store that throws an exception on non-initiator node. - */ - public void testBrokenCacheStoreOnNonInitiator() { - final boolean failOnAllNodes = false; - final int unluckyNode = 1; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = 2; - - testDynamicCacheStart( - createCacheConfigsWithBrokenCacheStore( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Tests cache start with broken cache store that throws an exception on initiator node. - */ - public void testBrokenCacheStoreOnCoordinatorDiffInitiator() { - final boolean failOnAllNodes = false; - final int unluckyNode = crdIdx; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = (crdIdx + 1) % gridCount(); - - testDynamicCacheStart( - createCacheConfigsWithBrokenCacheStore( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Tests cache start with broken cache store that throws an exception on coordinator node. - */ - public void testBrokenCacheStoreFunOnCoordinator() { - final boolean failOnAllNodes = false; - final int unluckyNode = crdIdx; - final int unluckyCfg = 1; - final int numOfCaches = 3; - final int initiator = crdIdx; - - testDynamicCacheStart( - createCacheConfigsWithBrokenCacheStore( - failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), - initiator); - } - - /** - * Tests multiple creation of cache with broken affinity function. - */ - public void testCreateCacheMultipleTimes() { - final boolean failOnAllNodes = false; - final int unluckyNode = 1; - final int unluckyCfg = 0; - final int numOfAttempts = 15; - - CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( - failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0); - - for (int i = 0; i < numOfAttempts; ++i) { - try { - IgniteCache cache = ignite(0).getOrCreateCache(cfg); - - fail("Expected exception was not thrown"); - } - catch (CacheException e) { - } - } - } - - /** - * Tests that a cache with the same name can be started after failure if cache configuration is corrected. - * - * @throws Exception If test failed. - */ - public void testCacheStartAfterFailure() throws Exception { - CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( - false, 1, 0, 1, false).get(0); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws Exception { - grid(0).getOrCreateCache(cfg); - return null; - } - }, CacheException.class, null); - - // Correct the cache configuration. Default constructor creates a good affinity function. - cfg.setAffinity(new BrokenAffinityFunction()); - - IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); - - checkCacheOperations(cache); - } - - /** - * Tests that other cache (existed before the failed start) is still operable after the failure. - * - * @throws Exception If test failed. - */ - public void testExistingCacheAfterFailure() throws Exception { - IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); - - CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( - false, 1, 0, 1, false).get(0); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws Exception { - grid(0).getOrCreateCache(cfg); - return null; - } - }, CacheException.class, null); - - checkCacheOperations(cache); - } - - /** - * Tests that other cache works as expected after the failure and further topology changes. - * - * @throws Exception If test failed. - */ - public void testTopologyChangesAfterFailure() throws Exception { - final String clientName = "testTopologyChangesAfterFailure"; - - IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); - - checkCacheOperations(cache); - - CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( - false, 0, 0, 1, false).get(0); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws Exception { - grid(0).getOrCreateCache(cfg); - return null; - } - }, CacheException.class, null); - - awaitPartitionMapExchange(); - - checkCacheOperations(cache); - - // Start a new server node and check cache operations. - Ignite serverNode = startGrid(gridCount() + 1); - - if (persistenceEnabled()) { - // Start a new client node to perform baseline change. - // TODO: This change is workaround: - // Sometimes problem with caches configuration deserialization from test thread arises. - final String clientName1 = "baseline-changer"; - - IgniteConfiguration clientCfg = getConfiguration(clientName1); - - clientCfg.setClientMode(true); - - Ignite clientNode = startGrid(clientName1, clientCfg); - - List baseline = new ArrayList<>(grid(0).cluster().currentBaselineTopology()); - - baseline.add(serverNode.cluster().localNode()); - - clientNode.cluster().setBaselineTopology(baseline); - } - - awaitPartitionMapExchange(); - - checkCacheOperations(serverNode.cache(EXISTING_CACHE_NAME)); - - // Start a new client node and check cache operations. - IgniteConfiguration clientCfg = getConfiguration(clientName); - - clientCfg.setClientMode(true); - - Ignite clientNode = startGrid(clientName, clientCfg); - - checkCacheOperations(clientNode.cache(EXISTING_CACHE_NAME)); - } - - public void testConcurrentClientNodeJoins() throws Exception { - final int clientCnt = 3; - final int numberOfAttempts = 5; - - IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); - - final AtomicInteger attemptCnt = new AtomicInteger(); - final CountDownLatch stopLatch = new CountDownLatch(clientCnt); - - IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable() { - @Override public Object call() throws Exception { - String clientName = Thread.currentThread().getName(); - - try { - for (int i = 0; i < numberOfAttempts; ++i) { - int uniqueCnt = attemptCnt.getAndIncrement(); - - IgniteConfiguration clientCfg = getConfiguration(clientName + uniqueCnt); - - clientCfg.setClientMode(true); - - final Ignite clientNode = startGrid(clientName, clientCfg); - - CacheConfiguration cfg = new CacheConfiguration(); - - cfg.setName(clientName + uniqueCnt); - - String instanceName = getTestIgniteInstanceName(uniqueCnt % gridCount()); - - cfg.setAffinity(new BrokenAffinityFunction(false, instanceName)); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws Exception { - clientNode.getOrCreateCache(cfg); - return null; - } - }, CacheException.class, null); - - stopGrid(clientName, true); - } - } - catch (Exception e) { - fail("Unexpected exception: " + e.getMessage()); - } - finally { - stopLatch.countDown(); - } - - return null; - } - }, clientCnt, "start-client-thread"); - - stopLatch.await(); - - assertEquals(numberOfAttempts * clientCnt, attemptCnt.get()); - - checkCacheOperations(cache); - } - - protected void testDynamicCacheStart(final Collection cfgs, final int initiatorId) { - assert initiatorId < gridCount(); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws Exception { - grid(initiatorId).getOrCreateCaches(cfgs); - return null; - } - }, CacheException.class, null); - - for (CacheConfiguration cfg: cfgs) { - IgniteCache cache = grid(initiatorId).cache(cfg.getName()); - - assertNull(cache); - } - } - - /** - * Creates new cache configuration with the given name. - * - * @param cacheName Cache name. - * @return New cache configuration. - */ - protected CacheConfiguration createCacheConfiguration(String cacheName) { - CacheConfiguration cfg = new CacheConfiguration() - .setName(cacheName) - .setCacheMode(CacheMode.PARTITIONED) - .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) - .setAffinity(new BrokenAffinityFunction()); - - return cfg; - } - - /** - * Create list of cache configurations. - * - * @param failOnAllNodes {@code true} if affinity function should be broken on all nodes. - * @param unluckyNode Node, where exception is raised. - * @param unluckyCfg Unlucky cache configuration number. - * @param cacheNum Number of caches. - * @param useFilter {@code true} if NodeFilter should be used. - * - * @return List of cache configurations. - */ - protected List createCacheConfigsWithBrokenAffinityFun( - boolean failOnAllNodes, - int unluckyNode, - final int unluckyCfg, - int cacheNum, - boolean useFilter - ) { - assert unluckyCfg >= 0 && unluckyCfg < cacheNum; - - final UUID uuid = ignite(unluckyNode).cluster().localNode().id(); - - List cfgs = new ArrayList<>(); - - for (int i = 0; i < cacheNum; ++i) { - CacheConfiguration cfg = createCacheConfiguration(DYNAMIC_CACHE_NAME + "-" + i); - - if (i == unluckyCfg) - cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestIgniteInstanceName(unluckyNode))); - - if (useFilter) - cfg.setNodeFilter(new NodeFilter(uuid)); - - cfgs.add(cfg); - } - - return cfgs; - } - - /** - * Create list of cache configurations. - * - * @param failOnAllNodes {@code true} if cache store should be broken on all nodes. - * @param unluckyNode Node, where exception is raised. - * @param unluckyCfg Unlucky cache configuration number. - * @param cacheNum Number of caches. - * @param useFilter {@code true} if NodeFilter should be used. - * - * @return List of cache configurations. - */ - protected List createCacheConfigsWithBrokenCacheStore( - boolean failOnAllNodes, - int unluckyNode, - int unluckyCfg, - int cacheNum, - boolean useFilter - ) { - assert unluckyCfg >= 0 && unluckyCfg < cacheNum; - - final UUID uuid = ignite(unluckyNode).cluster().localNode().id(); - - List cfgs = new ArrayList<>(); - - for (int i = 0; i < cacheNum; ++i) { - CacheConfiguration cfg = new CacheConfiguration(); - - cfg.setName(DYNAMIC_CACHE_NAME + "-" + i); - - if (i == unluckyCfg) - cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestIgniteInstanceName(unluckyNode))); - - if (useFilter) - cfg.setNodeFilter(new NodeFilter(uuid)); - - cfgs.add(cfg); - } - - return cfgs; - } - - /** - * Test the basic cache operations. - * - * @param cache Cache. - * @throws Exception If test failed. - */ - protected void checkCacheOperations(IgniteCache cache) throws Exception { - int cnt = 1000; - - // Check cache operations. - for (int i = 0; i < cnt; ++i) - cache.put(i, new Value(i)); - - for (int i = 0; i < cnt; ++i) { - Value v = cache.get(i); - - assertNotNull(v); - assertEquals(i, v.getValue()); - } - - // Check Data Streamer functionality. - try (IgniteDataStreamer streamer = grid(0).dataStreamer(cache.getName())) { - for (int i = 0; i < 10_000; ++i) - streamer.addData(i, new Value(i)); - } - } - - /** - * - */ - public static class Value { - @QuerySqlField - private final int fieldVal; - - public Value(int fieldVal) { - this.fieldVal = fieldVal; - } - - public int getValue() { - return fieldVal; - } - } - - /** - * Filter specifying on which node the cache should be started. - */ - public static class NodeFilter implements IgnitePredicate { - /** Cache should be created node with certain UUID. */ - public UUID uuid; - - /** - * @param uuid node ID. - */ - public NodeFilter(UUID uuid) { - this.uuid = uuid; - } - - /** {@inheritDoc} */ - @Override public boolean apply(ClusterNode clusterNode) { - return clusterNode.id().equals(uuid); - } - } - - /** - * Affinity function that throws an exception when affinity nodes are calculated on the given node. - */ - public static class BrokenAffinityFunction extends RendezvousAffinityFunction { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - @IgniteInstanceResource - private Ignite ignite; - - /** Exception should arise on all nodes. */ - private boolean eOnAllNodes = false; - - /** Exception should arise on node with certain name. */ - private String gridName; - - /** - * Constructs a good affinity function. - */ - public BrokenAffinityFunction() { - super(false, PARTITION_COUNT); - // No-op. - } - - /** - * @param eOnAllNodes {@code True} if exception should be thrown on all nodes. - * @param gridName Exception should arise on node with certain name. - */ - public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) { - super(false, PARTITION_COUNT); - - this.eOnAllNodes = eOnAllNodes; - this.gridName = gridName; - } - - /** {@inheritDoc} */ - @Override public List> assignPartitions(AffinityFunctionContext affCtx) { - if (eOnAllNodes || ignite.name().equals(gridName)) - throw new IllegalStateException("Simulated exception [locNodeId=" - + ignite.cluster().localNode().id() + "]"); - else - return super.assignPartitions(affCtx); - } - } - - /** - * Factory that throws an exception is got created. - */ - public static class BrokenStoreFactory implements Factory> { - /** */ - @IgniteInstanceResource - private Ignite ignite; - - /** Exception should arise on all nodes. */ - boolean eOnAllNodes = true; - - /** Exception should arise on node with certain name. */ - public static String gridName; - - /** - * @param eOnAllNodes {@code True} if exception should be thrown on all nodes. - * @param gridName Exception should arise on node with certain name. - */ - public BrokenStoreFactory(boolean eOnAllNodes, String gridName) { - this.eOnAllNodes = eOnAllNodes; - - this.gridName = gridName; - } - - /** {@inheritDoc} */ - @Override public CacheStore create() { - if (eOnAllNodes || ignite.name().equals(gridName)) - throw new IllegalStateException("Simulated exception [locNodeId=" - + ignite.cluster().localNode().id() + "]"); - else - return null; - } - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.ObjectInputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.CacheException; +import javax.cache.configuration.Factory; +import javax.management.Attribute; +import javax.management.AttributeList; +import javax.management.AttributeNotFoundException; +import javax.management.InstanceAlreadyExistsException; +import javax.management.InstanceNotFoundException; +import javax.management.IntrospectionException; +import javax.management.InvalidAttributeValueException; +import javax.management.ListenerNotFoundException; +import javax.management.MBeanException; +import javax.management.MBeanInfo; +import javax.management.MBeanRegistrationException; +import javax.management.MBeanServer; +import javax.management.NotCompliantMBeanException; +import javax.management.NotificationFilter; +import javax.management.NotificationListener; +import javax.management.ObjectInstance; +import javax.management.ObjectName; +import javax.management.OperationsException; +import javax.management.QueryExp; +import javax.management.ReflectionException; +import javax.management.loading.ClassLoaderRepository; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.AffinityFunctionContext; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.store.CacheStore; +import org.apache.ignite.cluster.BaselineNode; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.testframework.GridTestUtils; + +/** + * Tests the recovery after a dynamic cache start failure. + */ +public abstract class IgniteAbstractDynamicCacheStartFailTest extends GridCacheAbstractSelfTest { + /** */ + private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache"; + + /** */ + private static final String CLIENT_GRID_NAME = "client"; + + /** */ + protected static final String EXISTING_CACHE_NAME = "existing-cache";; + + /** */ + private static final int PARTITION_COUNT = 16; + + /** Failure MBean server. */ + private static FailureMBeanServer mbSrv; + + /** Coordinator node index. */ + private int crdIdx = 0; + + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 3; + } + + /** + * Returns {@code true} if persistence is enabled. + * + * @return {@code true} if persistence is enabled. + */ + protected boolean persistenceEnabled() { + return false; + } + + /** + * @throws Exception If failed. + */ + public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception { + final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient"; + + IgniteConfiguration clientCfg = getConfiguration(clientName); + + clientCfg.setClientMode(true); + + Ignite client = startGrid(clientName, clientCfg); + + CacheConfiguration cfg = new CacheConfiguration(); + + cfg.setName(DYNAMIC_CACHE_NAME + "-server-1"); + + cfg.setAffinity(new BrokenAffinityFunction(false, clientName)); + + try { + IgniteCache cache = ignite(0).getOrCreateCache(cfg); + } + catch (CacheException e) { + fail("Exception should not be thrown."); + } + + stopGrid(clientName); + } + + /** + * @throws Exception If failed. + */ + public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception { + final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer"; + + IgniteConfiguration clientCfg = getConfiguration(clientName); + + clientCfg.setClientMode(true); + + Ignite client = startGrid(clientName, clientCfg); + + CacheConfiguration cfg = new CacheConfiguration(); + + cfg.setName(DYNAMIC_CACHE_NAME + "-server-2"); + + cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0))); + + try { + IgniteCache cache = ignite(0).getOrCreateCache(cfg); + + fail("Expected exception was not thrown."); + } + catch (CacheException e) { + } + + stopGrid(clientName); + } + + /** + * @throws Exception If failed. + */ + public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception { + final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer"; + + IgniteConfiguration clientCfg = getConfiguration(clientName); + + clientCfg.setClientMode(true); + + Ignite client = startGrid(clientName, clientCfg); + + CacheConfiguration cfg = new CacheConfiguration(); + + cfg.setName(DYNAMIC_CACHE_NAME + "-client-2"); + + cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0))); + + try { + IgniteCache cache = client.getOrCreateCache(cfg); + + fail("Expected exception was not thrown."); + } + catch (CacheException e) { + } + + stopGrid(clientName); + } + + /** + * Test cache start with broken affinity function that throws an exception on all nodes. + */ + public void testBrokenAffinityFunOnAllNodes() { + final boolean failOnAllNodes = true; + final int unluckyNode = 0; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 0; + + testDynamicCacheStart( + createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Test cache start with broken affinity function that throws an exception on initiator node. + */ + public void testBrokenAffinityFunOnInitiator() { + final boolean failOnAllNodes = false; + final int unluckyNode = 1; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 1; + + testDynamicCacheStart( + createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Test cache start with broken affinity function that throws an exception on non-initiator node. + */ + public void testBrokenAffinityFunOnNonInitiator() { + final boolean failOnAllNodes = false; + final int unluckyNode = 1; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 2; + + testDynamicCacheStart( + createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Test cache start with broken affinity function that throws an exception on coordinator node. + */ + public void testBrokenAffinityFunOnCoordinatorDiffInitiator() { + final boolean failOnAllNodes = false; + final int unluckyNode = crdIdx; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = (crdIdx + 1) % gridCount(); + + testDynamicCacheStart( + createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Test cache start with broken affinity function that throws an exception on initiator node. + */ + public void testBrokenAffinityFunOnCoordinator() { + final boolean failOnAllNodes = false; + final int unluckyNode = crdIdx; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = crdIdx; + + testDynamicCacheStart( + createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Tests cache start with node filter and broken affinity function that throws an exception on initiator node. + */ + public void testBrokenAffinityFunWithNodeFilter() { + final boolean failOnAllNodes = false; + final int unluckyNode = 0; + final int unluckyCfg = 0; + final int numOfCaches = 1; + final int initiator = 0; + + testDynamicCacheStart( + createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true), + initiator); + } + + /** + * Tests cache start with broken cache store that throws an exception on all nodes. + */ + public void testBrokenCacheStoreOnAllNodes() { + final boolean failOnAllNodes = true; + final int unluckyNode = 0; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 0; + + testDynamicCacheStart( + createCacheConfigsWithBrokenCacheStore( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Tests cache start with broken cache store that throws an exception on initiator node. + */ + public void testBrokenCacheStoreOnInitiator() { + final boolean failOnAllNodes = false; + final int unluckyNode = 1; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 1; + + testDynamicCacheStart( + createCacheConfigsWithBrokenCacheStore( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Tests cache start that throws an Ignite checked exception on initiator node. + */ + public void testThrowsIgniteCheckedExceptionOnInitiator() { + final int unluckyNode = 1; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 1; + + testDynamicCacheStart( + createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches), + initiator); + } + + /** + * Tests cache start with broken cache store that throws an exception on non-initiator node. + */ + public void testBrokenCacheStoreOnNonInitiator() { + final boolean failOnAllNodes = false; + final int unluckyNode = 1; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 2; + + testDynamicCacheStart( + createCacheConfigsWithBrokenCacheStore( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Tests cache start that throws an Ignite checked exception on non-initiator node. + */ + public void testThrowsIgniteCheckedExceptionOnNonInitiator() { + final int unluckyNode = 1; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = 2; + + testDynamicCacheStart( + createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches), + initiator); + } + + /** + * Tests cache start with broken cache store that throws an exception on initiator node. + */ + public void testBrokenCacheStoreOnCoordinatorDiffInitiator() { + final boolean failOnAllNodes = false; + final int unluckyNode = crdIdx; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = (crdIdx + 1) % gridCount(); + + testDynamicCacheStart( + createCacheConfigsWithBrokenCacheStore( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Tests cache start that throws an Ignite checked exception on coordinator node + * that doesn't initiator node. + */ + public void testThrowsIgniteCheckedExceptionOnCoordinatorDiffInitiator() { + final int unluckyNode = crdIdx; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = (crdIdx + 1) % gridCount(); + + testDynamicCacheStart( + createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches), + initiator); + } + + /** + * Tests cache start with broken cache store that throws an exception on coordinator node. + */ + public void testBrokenCacheStoreFunOnCoordinator() { + final boolean failOnAllNodes = false; + final int unluckyNode = crdIdx; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = crdIdx; + + testDynamicCacheStart( + createCacheConfigsWithBrokenCacheStore( + failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false), + initiator); + } + + /** + * Tests cache start that throws an Ignite checked exception on coordinator node. + */ + public void testThrowsIgniteCheckedExceptionOnCoordinator() { + final int unluckyNode = crdIdx; + final int unluckyCfg = 1; + final int numOfCaches = 3; + final int initiator = crdIdx; + + testDynamicCacheStart( + createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches), + initiator); + } + + /** + * Tests multiple creation of cache with broken affinity function. + */ + public void testCreateCacheMultipleTimes() { + final boolean failOnAllNodes = false; + final int unluckyNode = 1; + final int unluckyCfg = 0; + final int numOfAttempts = 15; + + CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( + failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0); + + for (int i = 0; i < numOfAttempts; ++i) { + try { + IgniteCache cache = ignite(0).getOrCreateCache(cfg); + + fail("Expected exception was not thrown"); + } + catch (CacheException e) { + } + } + } + + /** + * Tests that a cache with the same name can be started after failure if cache configuration is corrected. + * + * @throws Exception If test failed. + */ + public void testCacheStartAfterFailure() throws Exception { + CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( + false, 1, 0, 1, false).get(0); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + grid(0).getOrCreateCache(cfg); + return null; + } + }, CacheException.class, null); + + // Correct the cache configuration. Default constructor creates a good affinity function. + cfg.setAffinity(new BrokenAffinityFunction()); + + checkCacheOperations(grid(0).getOrCreateCache(cfg)); + } + + /** + * Tests that other cache (existed before the failed start) is still operable after the failure. + * + * @throws Exception If test failed. + */ + public void testExistingCacheAfterFailure() throws Exception { + IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); + + CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( + false, 1, 0, 1, false).get(0); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + grid(0).getOrCreateCache(cfg); + return null; + } + }, CacheException.class, null); + + checkCacheOperations(cache); + } + + /** + * Tests that other cache works as expected after the failure and further topology changes. + * + * @throws Exception If test failed. + */ + public void testTopologyChangesAfterFailure() throws Exception { + final String clientName = "testTopologyChangesAfterFailure"; + + IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); + + checkCacheOperations(cache); + + CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun( + false, 0, 0, 1, false).get(0); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + grid(0).getOrCreateCache(cfg); + return null; + } + }, CacheException.class, null); + + awaitPartitionMapExchange(); + + checkCacheOperations(cache); + + // Start a new server node and check cache operations. + Ignite serverNode = startGrid(gridCount() + 1); + + if (persistenceEnabled()) { + // Start a new client node to perform baseline change. + // TODO: This change is workaround: + // Sometimes problem with caches configuration deserialization from test thread arises. + final String clientName1 = "baseline-changer"; + + IgniteConfiguration clientCfg = getConfiguration(clientName1); + + clientCfg.setClientMode(true); + + Ignite clientNode = startGrid(clientName1, clientCfg); + + List baseline = new ArrayList<>(grid(0).cluster().currentBaselineTopology()); + + baseline.add(serverNode.cluster().localNode()); + + clientNode.cluster().setBaselineTopology(baseline); + } + + awaitPartitionMapExchange(); + + checkCacheOperations(serverNode.cache(EXISTING_CACHE_NAME)); + + // Start a new client node and check cache operations. + IgniteConfiguration clientCfg = getConfiguration(clientName); + + clientCfg.setClientMode(true); + + Ignite clientNode = startGrid(clientName, clientCfg); + + checkCacheOperations(clientNode.cache(EXISTING_CACHE_NAME)); + } + + public void testConcurrentClientNodeJoins() throws Exception { + final int clientCnt = 3; + final int numberOfAttempts = 5; + + IgniteCache cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME)); + + final AtomicInteger attemptCnt = new AtomicInteger(); + final CountDownLatch stopLatch = new CountDownLatch(clientCnt); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable() { + @Override public Object call() throws Exception { + String clientName = Thread.currentThread().getName(); + + try { + for (int i = 0; i < numberOfAttempts; ++i) { + int uniqueCnt = attemptCnt.getAndIncrement(); + + IgniteConfiguration clientCfg = getConfiguration(clientName + uniqueCnt); + + clientCfg.setClientMode(true); + + final Ignite clientNode = startGrid(clientName, clientCfg); + + CacheConfiguration cfg = new CacheConfiguration(); + + cfg.setName(clientName + uniqueCnt); + + String instanceName = getTestIgniteInstanceName(uniqueCnt % gridCount()); + + cfg.setAffinity(new BrokenAffinityFunction(false, instanceName)); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + clientNode.getOrCreateCache(cfg); + return null; + } + }, CacheException.class, null); + + stopGrid(clientName, true); + } + } + catch (Exception e) { + fail("Unexpected exception: " + e.getMessage()); + } + finally { + stopLatch.countDown(); + } + + return null; + } + }, clientCnt, "start-client-thread"); + + stopLatch.await(); + + assertEquals(numberOfAttempts * clientCnt, attemptCnt.get()); + + checkCacheOperations(cache); + } + + protected void testDynamicCacheStart(final Collection cfgs, final int initiatorId) { + assert initiatorId < gridCount(); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + grid(initiatorId).getOrCreateCaches(cfgs); + return null; + } + }, CacheException.class, null); + + for (CacheConfiguration cfg: cfgs) { + IgniteCache cache = grid(initiatorId).cache(cfg.getName()); + + assertNull(cache); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration res = super.getConfiguration(igniteInstanceName); + + if (mbSrv == null) + mbSrv = new FailureMBeanServer(res.getMBeanServer()); + + res.setMBeanServer(mbSrv); + + return res; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + mbSrv.clear(); + + for (String cacheName : grid(0).cacheNames()) { + if (!(EXISTING_CACHE_NAME.equals(cacheName) || DEFAULT_CACHE_NAME.equals(cacheName))) + grid(0).cache(cacheName).destroy(); + } + } + + /** + * Creates new cache configuration with the given name. + * + * @param cacheName Cache name. + * @return New cache configuration. + */ + protected CacheConfiguration createCacheConfiguration(String cacheName) { + CacheConfiguration cfg = new CacheConfiguration() + .setName(cacheName) + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setAffinity(new BrokenAffinityFunction()); + + return cfg; + } + + /** + * Create list of cache configurations. + * + * @param failOnAllNodes {@code true} if affinity function should be broken on all nodes. + * @param unluckyNode Node, where exception is raised. + * @param unluckyCfg Unlucky cache configuration number. + * @param cacheNum Number of caches. + * @param useFilter {@code true} if NodeFilter should be used. + * + * @return List of cache configurations. + */ + protected List createCacheConfigsWithBrokenAffinityFun( + boolean failOnAllNodes, + int unluckyNode, + final int unluckyCfg, + int cacheNum, + boolean useFilter + ) { + assert unluckyCfg >= 0 && unluckyCfg < cacheNum; + + final UUID uuid = ignite(unluckyNode).cluster().localNode().id(); + + List cfgs = new ArrayList<>(); + + for (int i = 0; i < cacheNum; ++i) { + CacheConfiguration cfg = createCacheConfiguration(DYNAMIC_CACHE_NAME + "-" + i); + + if (i == unluckyCfg) + cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestIgniteInstanceName(unluckyNode))); + + if (useFilter) + cfg.setNodeFilter(new NodeFilter(uuid)); + + cfgs.add(cfg); + } + + return cfgs; + } + + /** + * Create list of cache configurations. + * + * @param failOnAllNodes {@code true} if cache store should be broken on all nodes. + * @param unluckyNode Node, where exception is raised. + * @param unluckyCfg Unlucky cache configuration number. + * @param cacheNum Number of caches. + * @param useFilter {@code true} if NodeFilter should be used. + * + * @return List of cache configurations. + */ + protected List createCacheConfigsWithBrokenCacheStore( + boolean failOnAllNodes, + int unluckyNode, + int unluckyCfg, + int cacheNum, + boolean useFilter + ) { + assert unluckyCfg >= 0 && unluckyCfg < cacheNum; + + final UUID uuid = ignite(unluckyNode).cluster().localNode().id(); + + List cfgs = new ArrayList<>(); + + for (int i = 0; i < cacheNum; ++i) { + CacheConfiguration cfg = new CacheConfiguration(); + + cfg.setName(DYNAMIC_CACHE_NAME + "-" + i); + + if (i == unluckyCfg) + cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestIgniteInstanceName(unluckyNode))); + + if (useFilter) + cfg.setNodeFilter(new NodeFilter(uuid)); + + cfgs.add(cfg); + } + + return cfgs; + } + + /** + * Create list of cache configurations. + * + * @param unluckyNode Node, where exception is raised. + * @param unluckyCfg Unlucky cache configuration number. + * @param cacheNum Number of caches. + * + * @return List of cache configurations. + */ + private List createCacheConfigsWithFailureMbServer( + int unluckyNode, + int unluckyCfg, + int cacheNum + ) { + assert unluckyCfg >= 0 && unluckyCfg < cacheNum; + + List cfgs = new ArrayList<>(); + + for (int i = 0; i < cacheNum; ++i) { + CacheConfiguration cfg = new CacheConfiguration(); + + String cacheName = DYNAMIC_CACHE_NAME + "-" + i; + + cfg.setName(cacheName); + + if (i == unluckyCfg) + mbSrv.cache(cacheName); + + cfgs.add(cfg); + } + + mbSrv.node(getTestIgniteInstanceName(unluckyNode)); + + return cfgs; + } + + /** + * Test the basic cache operations. + * + * @param cache Cache. + * @throws Exception If test failed. + */ + protected void checkCacheOperations(IgniteCache cache) throws Exception { + int cnt = 1000; + + // Check cache operations. + for (int i = 0; i < cnt; ++i) + cache.put(i, new Value(i)); + + for (int i = 0; i < cnt; ++i) { + Value v = cache.get(i); + + assertNotNull(v); + assertEquals(i, v.getValue()); + } + + // Check Data Streamer functionality. + try (IgniteDataStreamer streamer = grid(0).dataStreamer(cache.getName())) { + for (int i = 0; i < 10_000; ++i) + streamer.addData(i, new Value(i)); + } + } + + /** + * + */ + public static class Value { + @QuerySqlField + private final int fieldVal; + + public Value(int fieldVal) { + this.fieldVal = fieldVal; + } + + public int getValue() { + return fieldVal; + } + } + + /** + * Filter specifying on which node the cache should be started. + */ + public static class NodeFilter implements IgnitePredicate { + /** Cache should be created node with certain UUID. */ + public UUID uuid; + + /** + * @param uuid node ID. + */ + public NodeFilter(UUID uuid) { + this.uuid = uuid; + } + + /** {@inheritDoc} */ + @Override public boolean apply(ClusterNode clusterNode) { + return clusterNode.id().equals(uuid); + } + } + + /** + * Affinity function that throws an exception when affinity nodes are calculated on the given node. + */ + public static class BrokenAffinityFunction extends RendezvousAffinityFunction { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + @IgniteInstanceResource + private Ignite ignite; + + /** Exception should arise on all nodes. */ + private boolean eOnAllNodes = false; + + /** Exception should arise on node with certain name. */ + private String gridName; + + /** + * Constructs a good affinity function. + */ + public BrokenAffinityFunction() { + super(false, PARTITION_COUNT); + // No-op. + } + + /** + * @param eOnAllNodes {@code True} if exception should be thrown on all nodes. + * @param gridName Exception should arise on node with certain name. + */ + public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) { + super(false, PARTITION_COUNT); + + this.eOnAllNodes = eOnAllNodes; + this.gridName = gridName; + } + + /** {@inheritDoc} */ + @Override public List> assignPartitions(AffinityFunctionContext affCtx) { + if (eOnAllNodes || ignite.name().equals(gridName)) + throw new IllegalStateException("Simulated exception [locNodeId=" + + ignite.cluster().localNode().id() + "]"); + else + return super.assignPartitions(affCtx); + } + } + + /** + * Factory that throws an exception is got created. + */ + public static class BrokenStoreFactory implements Factory> { + /** */ + @IgniteInstanceResource + private Ignite ignite; + + /** Exception should arise on all nodes. */ + boolean eOnAllNodes = true; + + /** Exception should arise on node with certain name. */ + public static String gridName; + + /** + * @param eOnAllNodes {@code True} if exception should be thrown on all nodes. + * @param gridName Exception should arise on node with certain name. + */ + public BrokenStoreFactory(boolean eOnAllNodes, String gridName) { + this.eOnAllNodes = eOnAllNodes; + + this.gridName = gridName; + } + + /** {@inheritDoc} */ + @Override public CacheStore create() { + if (eOnAllNodes || ignite.name().equals(gridName)) + throw new IllegalStateException("Simulated exception [locNodeId=" + + ignite.cluster().localNode().id() + "]"); + else + return null; + } + } + + /** Failure MBean server. */ + private class FailureMBeanServer implements MBeanServer { + /** */ + private final MBeanServer origin; + + /** Set of caches that must be failure. */ + private final Set caches = new HashSet<>(); + + /** Set of nodes that must be failure. */ + private final Set nodes = new HashSet<>(); + + /** */ + private FailureMBeanServer(MBeanServer origin) { + this.origin = origin; + } + + /** Add cache name to failure set. */ + void cache(String cache) { + caches.add('\"' + cache + '\"'); + } + + /** Add node name to failure set. */ + void node(String node) { + nodes.add(node); + } + + /** Clear failure set of caches and set of nodes. */ + void clear() { + caches.clear(); + nodes.clear(); + } + + /** {@inheritDoc} */ + @Override public ObjectInstance registerMBean(Object obj, ObjectName name) + throws InstanceAlreadyExistsException, MBeanRegistrationException, NotCompliantMBeanException { + String node = name.getKeyProperty("igniteInstanceName"); + + if (nodes.contains(node) && caches.contains(name.getKeyProperty("group"))) + throw new MBeanRegistrationException(new Exception("Simulate exception [node=" + node + ']')); + + return origin.registerMBean(obj, name); + } + + /** {@inheritDoc} */ + @Override public ObjectInstance createMBean(String clsName, ObjectName name) + throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException { + return origin.createMBean(clsName, name); + } + + /** {@inheritDoc} */ + @Override public ObjectInstance createMBean(String clsName, ObjectName name, ObjectName ldrName) + throws ReflectionException, InstanceAlreadyExistsException, + MBeanException, NotCompliantMBeanException, InstanceNotFoundException { + return origin.createMBean(clsName, name, ldrName); + } + + /** {@inheritDoc} */ + @Override public ObjectInstance createMBean(String clsName, ObjectName name, Object[] params, + String[] signature) throws ReflectionException, InstanceAlreadyExistsException, + MBeanException, NotCompliantMBeanException { + return origin.createMBean(clsName, name, params, signature); + } + + /** {@inheritDoc} */ + @Override public ObjectInstance createMBean(String clsName, ObjectName name, ObjectName ldrName, + Object[] params, String[] signature) throws ReflectionException, InstanceAlreadyExistsException, + MBeanException, NotCompliantMBeanException, InstanceNotFoundException { + return origin.createMBean(clsName, name, ldrName, params, signature); + } + + /** {@inheritDoc} */ + @Override public void unregisterMBean(ObjectName name) throws InstanceNotFoundException, MBeanRegistrationException { + origin.unregisterMBean(name); + } + + /** {@inheritDoc} */ + @Override public ObjectInstance getObjectInstance(ObjectName name) throws InstanceNotFoundException { + return origin.getObjectInstance(name); + } + + /** {@inheritDoc} */ + @Override public Set queryMBeans(ObjectName name, QueryExp qry) { + return origin.queryMBeans(name, qry); + } + + /** {@inheritDoc} */ + @Override public Set queryNames(ObjectName name, QueryExp qry) { + return origin.queryNames(name, qry); + } + + /** {@inheritDoc} */ + @Override public boolean isRegistered(ObjectName name) { + return origin.isRegistered(name); + } + + /** {@inheritDoc} */ + @Override public Integer getMBeanCount() { + return origin.getMBeanCount(); + } + + /** {@inheritDoc} */ + @Override public Object getAttribute(ObjectName name, String attribute) + throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException { + return origin.getAttribute(name, attribute); + } + + /** {@inheritDoc} */ + @Override public AttributeList getAttributes(ObjectName name, + String[] attrs) throws InstanceNotFoundException, ReflectionException { + return origin.getAttributes(name, attrs); + } + + /** {@inheritDoc} */ + @Override public void setAttribute(ObjectName name, + Attribute attribute) throws InstanceNotFoundException, AttributeNotFoundException, + InvalidAttributeValueException, MBeanException, ReflectionException { + origin.setAttribute(name, attribute); + } + + /** {@inheritDoc} */ + @Override public AttributeList setAttributes(ObjectName name, + AttributeList attrs) throws InstanceNotFoundException, ReflectionException { + return origin.setAttributes(name, attrs); + } + + /** {@inheritDoc} */ + @Override public Object invoke(ObjectName name, String operationName, Object[] params, + String[] signature) throws InstanceNotFoundException, MBeanException, ReflectionException { + return origin.invoke(name, operationName, params, signature); + } + + /** {@inheritDoc} */ + @Override public String getDefaultDomain() { + return origin.getDefaultDomain(); + } + + /** {@inheritDoc} */ + @Override public String[] getDomains() { + return origin.getDomains(); + } + + /** {@inheritDoc} */ + @Override public void addNotificationListener(ObjectName name, NotificationListener lsnr, + NotificationFilter filter, Object handback) throws InstanceNotFoundException { + origin.addNotificationListener(name, lsnr, filter, handback); + } + + /** {@inheritDoc} */ + @Override public void addNotificationListener(ObjectName name, ObjectName lsnr, + NotificationFilter filter, Object handback) throws InstanceNotFoundException { + origin.addNotificationListener(name, lsnr, filter, handback); + } + + /** {@inheritDoc} */ + @Override public void removeNotificationListener(ObjectName name, + ObjectName lsnr) throws InstanceNotFoundException, ListenerNotFoundException { + origin.removeNotificationListener(name, lsnr); + } + + /** {@inheritDoc} */ + @Override public void removeNotificationListener(ObjectName name, ObjectName lsnr, + NotificationFilter filter, Object handback) throws InstanceNotFoundException, ListenerNotFoundException { + origin.removeNotificationListener(name, lsnr, filter, handback); + } + + /** {@inheritDoc} */ + @Override public void removeNotificationListener(ObjectName name, + NotificationListener lsnr) throws InstanceNotFoundException, ListenerNotFoundException { + origin.removeNotificationListener(name, lsnr); + } + + /** {@inheritDoc} */ + @Override public void removeNotificationListener(ObjectName name, NotificationListener lsnr, + NotificationFilter filter, Object handback) throws InstanceNotFoundException, ListenerNotFoundException { + origin.removeNotificationListener(name, lsnr, filter, handback); + } + + /** {@inheritDoc} */ + @Override public MBeanInfo getMBeanInfo( + ObjectName name) throws InstanceNotFoundException, IntrospectionException, ReflectionException { + return origin.getMBeanInfo(name); + } + + /** {@inheritDoc} */ + @Override public boolean isInstanceOf(ObjectName name, String clsName) throws InstanceNotFoundException { + return origin.isInstanceOf(name, clsName); + } + + /** {@inheritDoc} */ + @Override public Object instantiate(String clsName) throws ReflectionException, MBeanException { + return origin.instantiate(clsName); + } + + /** {@inheritDoc} */ + @Override public Object instantiate(String clsName, + ObjectName ldrName) throws ReflectionException, MBeanException, InstanceNotFoundException { + return origin.instantiate(clsName, ldrName); + } + + /** {@inheritDoc} */ + @Override public Object instantiate(String clsName, Object[] params, + String[] signature) throws ReflectionException, MBeanException { + return origin.instantiate(clsName, params, signature); + } + + /** {@inheritDoc} */ + @Override public Object instantiate(String clsName, ObjectName ldrName, Object[] params, + String[] signature) throws ReflectionException, MBeanException, InstanceNotFoundException { + return origin.instantiate(clsName, ldrName, params, signature); + } + + /** {@inheritDoc} */ + @Override @Deprecated public ObjectInputStream deserialize(ObjectName name, byte[] data) + throws OperationsException { + return origin.deserialize(name, data); + } + + /** {@inheritDoc} */ + @Override @Deprecated public ObjectInputStream deserialize(String clsName, byte[] data) + throws OperationsException, ReflectionException { + return origin.deserialize(clsName, data); + } + + /** {@inheritDoc} */ + @Override @Deprecated public ObjectInputStream deserialize(String clsName, ObjectName ldrName, byte[] data) + throws OperationsException, ReflectionException { + return origin.deserialize(clsName, ldrName, data); + } + + /** {@inheritDoc} */ + @Override public ClassLoader getClassLoaderFor(ObjectName mbeanName) throws InstanceNotFoundException { + return origin.getClassLoaderFor(mbeanName); + } + + /** {@inheritDoc} */ + @Override public ClassLoader getClassLoader(ObjectName ldrName) throws InstanceNotFoundException { + return origin.getClassLoader(ldrName); + } + + /** {@inheritDoc} */ + @Override public ClassLoaderRepository getClassLoaderRepository() { + return origin.getClassLoaderRepository(); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java index c67765651753c..f8a81ed606fcd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java @@ -285,6 +285,41 @@ public void testTransactionalSnapshotLimitations() throws Exception { ); } + /** + * Checks if passed in {@code 'Throwable'} has given class in {@code 'cause'} hierarchy + * including that throwable itself and it contains passed message. + *

+ * Note that this method follows includes {@link Throwable#getSuppressed()} + * into check. + * + * @param t Throwable to check (if {@code null}, {@code false} is returned). + * @param cls Cause class to check (if {@code null}, {@code false} is returned). + * @param msg Message to check. + * @return {@code True} if one of the causing exception is an instance of passed in classes + * and it contains the passed message, {@code false} otherwise. + */ + private boolean hasCauseWithMessage(@Nullable Throwable t, Class cls, String msg) { + if (t == null) + return false; + + assert cls != null; + + for (Throwable th = t; th != null; th = th.getCause()) { + if (cls.isAssignableFrom(th.getClass()) && th.getMessage() != null && th.getMessage().contains(msg)) + return true; + + for (Throwable n : th.getSuppressed()) { + if (hasCauseWithMessage(n, cls, msg)) + return true; + } + + if (th.getCause() == th) + break; + } + + return false; + } + /** * Make sure cache cannot be started with the given configuration. * @@ -305,7 +340,7 @@ private void assertCannotStart(CacheConfiguration ccfg, String msg) throws Excep catch (Exception e) { if (msg != null) { assert e.getMessage() != null : "Error message is null"; - assert e.getMessage().contains(msg) : "Wrong error message: " + e.getMessage(); + assertTrue(hasCauseWithMessage(e, IgniteCheckedException.class, msg)); } } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java index e56f8a2069b56..a91311a59699a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java @@ -17,16 +17,19 @@ package org.apache.ignite.internal.processors.query; +import java.util.concurrent.Callable; +import java.util.function.Consumer; +import javax.cache.CacheException; import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -import javax.cache.CacheException; -import java.util.concurrent.Callable; +import org.jetbrains.annotations.Nullable; /** * Tests for illegal SQL schemas in node and cache configurations. @@ -61,17 +64,27 @@ public void testBadCacheName() throws Exception { * @throws Exception If failed. */ public void testBadCacheNameDynamic() throws Exception { - Ignite node = startGrid(); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Void call() throws Exception { - node.getOrCreateCache(new CacheConfiguration().setName(QueryUtils.SCHEMA_SYS)); - - return null; + doubleConsumerAccept( + (node)->{ + try { + node.getOrCreateCache(new CacheConfiguration().setName(QueryUtils.SCHEMA_SYS)); + } + catch (CacheException e) { + assertTrue(hasCause(e, IgniteCheckedException.class, + "SQL schema name derived from cache name is reserved (please set explicit SQL " + + "schema name through CacheConfiguration.setSqlSchema() or choose another cache name) [" + + "cacheName=IGNITE, schemaName=null]")); + + return; + } + catch (Throwable e) { + fail("Exception class is not as expected [expected=" + + CacheException.class + ", actual=" + e.getClass() + ']'); + } + + fail("Exception has not been thrown."); } - }, CacheException.class, "SQL schema name derived from cache name is reserved (please set explicit SQL " + - "schema name through CacheConfiguration.setSqlSchema() or choose another cache name) [" + - "cacheName=IGNITE, schemaName=null]"); + ); } /** @@ -97,17 +110,27 @@ public void testBadSchemaLower() throws Exception { * @throws Exception If failed. */ public void testBadSchemaLowerDynamic() throws Exception { - Ignite node = startGrid(); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Void call() throws Exception { - node.getOrCreateCache( - new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toLowerCase()) - ); - - return null; + doubleConsumerAccept( + (node) -> { + try { + node.getOrCreateCache( + new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toLowerCase()) + ); + } + catch (CacheException e) { + assertTrue(hasCause(e, IgniteCheckedException.class, + "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=ignite]")); + + return; + } + catch (Throwable e) { + fail("Exception class is not as expected [expected=" + + CacheException.class + ", actual=" + e.getClass() + ']'); + } + + fail("Exception has not been thrown."); } - }, CacheException.class, "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=ignite]"); + ); } /** @@ -133,18 +156,27 @@ public void testBadSchemaUpper() throws Exception { * @throws Exception If failed. */ public void testBadSchemaUpperDynamic() throws Exception { - Ignite node = startGrid(); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Void call() throws Exception { - node.getOrCreateCache( - new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toUpperCase()) - ); - - return null; + doubleConsumerAccept( + (node) -> { + try { + node.getOrCreateCache( + new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toUpperCase()) + ); + } + catch (CacheException e) { + assertTrue(hasCause(e, IgniteCheckedException.class, + "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=IGNITE]")); + + return; + } + catch (Throwable e) { + fail("Exception class is not as expected [expected=" + + CacheException.class + ", actual=" + e.getClass() + ']'); + } + + fail("Exception has not been thrown."); } - }, CacheException.class, "SQL schema name is reserved (please choose another one) [cacheName=CACHE, " + - "schemaName=IGNITE]"); + ); } /** @@ -170,18 +202,76 @@ public void testBadSchemaQuoted() throws Exception { * @throws Exception If failed. */ public void testBadSchemaQuotedDynamic() throws Exception { + doubleConsumerAccept( + (node) -> { + try { + node.getOrCreateCache( + new CacheConfiguration().setName("CACHE") + .setSqlSchema("\"" + QueryUtils.SCHEMA_SYS.toUpperCase() + "\"") + ); + } + catch (CacheException e) { + assertTrue(hasCause(e, IgniteCheckedException.class, + "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=\"IGNITE\"]")); + + return; + } + catch (Throwable e) { + fail("Exception class is not as expected [expected=" + + CacheException.class + ", actual=" + e.getClass() + ']'); + } + + fail("Exception has not been thrown."); + } + ); + } + + /** + * Executes double call of consumer's accept method with passed Ignite instance. + * + * @param cons Consumer. + * @throws Exception If failed. + */ + private void doubleConsumerAccept(Consumer cons) throws Exception { Ignite node = startGrid(); - GridTestUtils.assertThrows(log, new Callable() { - @Override public Void call() throws Exception { - node.getOrCreateCache( - new CacheConfiguration().setName("CACHE") - .setSqlSchema("\"" + QueryUtils.SCHEMA_SYS.toUpperCase() + "\"") - ); + cons.accept(node); - return null; + cons.accept(node); + } + + /** + * Checks if passed in {@code 'Throwable'} has given class in {@code 'cause'} hierarchy + * including that throwable itself and it contains passed message. + *

+ * Note that this method follows includes {@link Throwable#getSuppressed()} + * into check. + * + * @param t Throwable to check (if {@code null}, {@code false} is returned). + * @param cls Cause class to check (if {@code null}, {@code false} is returned). + * @param msg Message to check. + * @return {@code True} if one of the causing exception is an instance of passed in classes + * and it contains the passed message, {@code false} otherwise. + */ + private boolean hasCause(@Nullable Throwable t, Class cls, String msg) { + if (t == null) + return false; + + assert cls != null; + + for (Throwable th = t; th != null; th = th.getCause()) { + if (cls.isAssignableFrom(th.getClass()) && F.eq(th.getMessage(), msg)) + return true; + + for (Throwable n : th.getSuppressed()) { + if (hasCause(n, cls, msg)) + return true; } - }, CacheException.class, "SQL schema name is reserved (please choose another one) [cacheName=CACHE, " + - "schemaName=\"IGNITE\"]"); + + if (th.getCause() == th) + break; + } + + return false; } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs index 7c447b180942b..a8aff71c63804 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs @@ -71,6 +71,7 @@ public void TearDown() /// Tests /// [Test] + [Ignore("IGNITE-10364")] public void TestStopNodeFailureHandler() { TestFailureHandler(typeof(StopNodeFailureHandler)); @@ -80,6 +81,7 @@ public void TestStopNodeFailureHandler() /// Tests /// [Test] + [Ignore("IGNITE-10364")] public void TestStopNodeOrHaltFailureHandler() { TestFailureHandler(typeof(StopNodeOrHaltFailureHandler)); From 67217d705cc9eb1b00b3e7c3280aa8355f5dbc05 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 29 Nov 2018 12:59:43 +0300 Subject: [PATCH 187/403] IGNITE-10435: Remove some dead code from thin clients. This closes #5521. --- .../platform/client/ClientRequest.java | 29 ----------------- .../cache/ClientCacheClearKeyRequest.java | 3 -- .../cache/ClientCacheClearKeysRequest.java | 3 -- .../client/cache/ClientCacheClearRequest.java | 3 -- .../cache/ClientCacheContainsKeyRequest.java | 3 -- .../cache/ClientCacheContainsKeysRequest.java | 3 -- ...ntCacheCreateWithConfigurationRequest.java | 9 ++---- .../ClientCacheCreateWithNameRequest.java | 6 ++-- .../cache/ClientCacheDestroyRequest.java | 3 -- .../cache/ClientCacheGetAllRequest.java | 5 +-- .../ClientCacheGetAndPutIfAbsentRequest.java | 3 -- .../cache/ClientCacheGetAndPutRequest.java | 3 -- .../cache/ClientCacheGetAndRemoveRequest.java | 3 -- .../ClientCacheGetAndReplaceRequest.java | 3 -- ...heGetOrCreateWithConfigurationRequest.java | 9 ++---- ...ClientCacheGetOrCreateWithNameRequest.java | 3 -- .../client/cache/ClientCacheGetRequest.java | 3 -- .../cache/ClientCacheGetSizeRequest.java | 3 -- .../cache/ClientCacheLocalPeekRequest.java | 3 -- .../ClientCacheNodePartitionsRequest.java | 2 -- .../cache/ClientCachePutAllRequest.java | 3 -- .../cache/ClientCachePutIfAbsentRequest.java | 3 -- .../client/cache/ClientCachePutRequest.java | 3 -- .../cache/ClientCacheRemoveAllRequest.java | 3 -- .../ClientCacheRemoveIfEqualsRequest.java | 3 -- .../cache/ClientCacheRemoveKeyRequest.java | 3 -- .../cache/ClientCacheRemoveKeysRequest.java | 3 -- .../ClientCacheReplaceIfEqualsRequest.java | 3 -- .../cache/ClientCacheReplaceRequest.java | 3 -- .../client/cache/ClientCacheRequest.java | 32 ------------------- .../cache/ClientCacheScanQueryRequest.java | 3 -- 31 files changed, 9 insertions(+), 155 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java index a4000efc009da..76823b592ce32 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java @@ -19,9 +19,6 @@ import org.apache.ignite.binary.BinaryRawReader; import org.apache.ignite.internal.processors.odbc.ClientListenerRequest; -import org.apache.ignite.internal.processors.security.SecurityContext; -import org.apache.ignite.plugin.security.SecurityException; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Thin client request. @@ -61,30 +58,4 @@ public ClientRequest(long reqId) { public ClientResponse process(ClientConnectionContext ctx) { return new ClientResponse(reqId); } - - /** - * Run the code with converting {@link SecurityException} to {@link IgniteClientException}. - */ - protected static void runWithSecurityExceptionHandler(Runnable runnable) { - try { - runnable.run(); - } - catch (SecurityException ex) { - throw new IgniteClientException( - ClientStatus.SECURITY_VIOLATION, - "Client is not authorized to perform this operation", - ex - ); - } - } - - /** - * Authorize for specified permission. - */ - protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) { -// SecurityContext secCtx = ctx.securityContext(); -// -// if (secCtx != null) -// runWithSecurityExceptionHandler(() -> ctx.kernalContext().security().authorize(null, perm, secCtx)); - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java index 5f8e952234bf8..6bcbbe89b2636 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java @@ -20,7 +20,6 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Clear key request. @@ -38,8 +37,6 @@ public ClientCacheClearKeyRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_REMOVE); - cache(ctx).clear(key()); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java index d803f697420f3..04eb7f60c9502 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java @@ -20,7 +20,6 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Clear keys request. @@ -38,8 +37,6 @@ public ClientCacheClearKeysRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_REMOVE); - cache(ctx).clearAll(keys()); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java index 7b84522921c7c..0e5f20de1eb1b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java @@ -20,7 +20,6 @@ import org.apache.ignite.binary.BinaryRawReader; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache clear request. @@ -38,8 +37,6 @@ public ClientCacheClearRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_REMOVE); - cache(ctx).clear(); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java index 386f448bb4b53..8470828e424a1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * ContainsKey request. @@ -39,8 +38,6 @@ public ClientCacheContainsKeyRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - boolean val = cache(ctx).containsKey(key()); return new ClientBooleanResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java index b5184bfc1afc9..41e13068db1f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * ContainsKeys request. @@ -39,8 +38,6 @@ public ClientCacheContainsKeysRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - boolean val = cache(ctx).containsKeys(keys()); return new ClientBooleanResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java index 9f1d63fccabc2..93a18edc77e9d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java @@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientResponse; import org.apache.ignite.internal.processors.platform.client.ClientStatus; import org.apache.ignite.internal.processors.platform.client.IgniteClientException; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache create with configuration request. @@ -50,12 +49,10 @@ public ClientCacheCreateWithConfigurationRequest(BinaryRawReader reader, ClientL /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_CREATE); - try { - // Use security exception handler since the code authorizes "enable on-heap cache" permission - runWithSecurityExceptionHandler(() -> ctx.kernalContext().grid().createCache(cacheCfg)); - } catch (CacheExistsException e) { + ctx.kernalContext().grid().createCache(cacheCfg); + } + catch (CacheExistsException e) { throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java index cacf099b4f81f..526eb6cdeb01a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java @@ -24,7 +24,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientResponse; import org.apache.ignite.internal.processors.platform.client.ClientStatus; import org.apache.ignite.internal.processors.platform.client.IgniteClientException; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache create with name request. @@ -46,11 +45,10 @@ public ClientCacheCreateWithNameRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_CREATE); - try { ctx.kernalContext().grid().createCache(cacheName); - } catch (CacheExistsException e) { + } + catch (CacheExistsException e) { throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java index b6f85eec3d9fc..6645a03a06b4f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientRequest; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache destroy request. @@ -43,8 +42,6 @@ public ClientCacheDestroyRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_DESTROY); - String cacheName = ClientCacheRequest.cacheDescriptor(ctx, cacheId).cacheName(); ctx.kernalContext().grid().destroyCache(cacheName); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java index a07305c4ce14b..d07da1cf46cb4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java @@ -22,7 +22,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientResponse; import java.util.Map; -import org.apache.ignite.plugin.security.SecurityPermission; /** * GetAll request. @@ -40,9 +39,7 @@ public ClientCacheGetAllRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - - Map val = cache(ctx).getAll(keys()); + Map val = cache(ctx).getAll(keys()); return new ClientCacheGetAllResponse(requestId(), val); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java index 8713a211bb4eb..836021313c5fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get and put if absent request. @@ -39,8 +38,6 @@ public ClientCacheGetAndPutIfAbsentRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT); - Object res = cache(ctx).getAndPutIfAbsent(key(), val()); return new ClientObjectResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java index dde5181303cef..7a540e8473ac9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get and put request. @@ -39,8 +38,6 @@ public ClientCacheGetAndPutRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT); - Object res = cache(ctx).getAndPut(key(), val()); return new ClientObjectResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java index 3b9dd4bab88c3..e4fd735b186ac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get and remove request. @@ -39,8 +38,6 @@ public ClientCacheGetAndRemoveRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_REMOVE); - Object val = cache(ctx).getAndRemove(key()); return new ClientObjectResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java index 8ba157a762c9c..dba8639e4c07a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get and replace request. @@ -39,8 +38,6 @@ public ClientCacheGetAndReplaceRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT); - Object res = cache(ctx).getAndReplace(key(), val()); return new ClientObjectResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java index b005fb235cf9f..77368beed7695 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java @@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientResponse; import org.apache.ignite.internal.processors.platform.client.ClientStatus; import org.apache.ignite.internal.processors.platform.client.IgniteClientException; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get or create with configuration request. @@ -50,12 +49,10 @@ public ClientCacheGetOrCreateWithConfigurationRequest(BinaryRawReader reader, Cl /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_CREATE); - try { - // Use security exception handler since the code authorizes "enable on-heap cache" permission - runWithSecurityExceptionHandler(() -> ctx.kernalContext().grid().getOrCreateCache(cacheCfg)); - } catch (CacheExistsException e) { + ctx.kernalContext().grid().getOrCreateCache(cacheCfg); + } + catch (CacheExistsException e) { throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java index 3c4ce7b06a694..94dd115d6075f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientRequest; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache create with name request. @@ -43,8 +42,6 @@ public ClientCacheGetOrCreateWithNameRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_CREATE); - ctx.kernalContext().grid().getOrCreateCache(cacheName); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java index dc17cbfbce548..41558c2863d03 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get request. @@ -39,8 +38,6 @@ public ClientCacheGetRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - Object val = cache(ctx).get(key()); return new ClientObjectResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java index 474c206b8cae3..ba185bf7415d8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java @@ -22,7 +22,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientLongResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache size request. @@ -51,8 +50,6 @@ public ClientCacheGetSizeRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - long res = cache(ctx).sizeLong(modes); return new ClientLongResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java index 068bbc9792c18..2002664d00714 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java @@ -22,7 +22,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache local peek request. @@ -41,8 +40,6 @@ public ClientCacheLocalPeekRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - Object val = cache(ctx).localPeek(key(), CachePeekMode.ALL); return new ClientObjectResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java index 377d26fae039a..abc9acf646ba1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheNodePartitionsRequest.java @@ -29,7 +29,6 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerProcessor; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cluster node list request. @@ -46,7 +45,6 @@ public ClientCacheNodePartitionsRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); IgniteCache cache = cache(ctx); GridDiscoveryManager discovery = ctx.kernalContext().discovery(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java index 57e31443b474a..28a7fa57e3ee5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java @@ -23,7 +23,6 @@ import java.util.LinkedHashMap; import java.util.Map; -import org.apache.ignite.plugin.security.SecurityPermission; /** * PutAll request. @@ -51,8 +50,6 @@ public ClientCachePutAllRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_PUT); - cache(ctx).putAll(map); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java index ec81bc0c0fe48..4dd2cde58ce06 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache put if absent request. @@ -39,8 +38,6 @@ public ClientCachePutIfAbsentRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT); - boolean res = cache(ctx).putIfAbsent(key(), val()); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java index 116460eece965..2c396b7ede87a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java @@ -20,7 +20,6 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache put request. @@ -38,8 +37,6 @@ public ClientCachePutRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_PUT); - cache(ctx).put(key(), val()); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java index d90d873968105..f5adc6378912e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java @@ -20,7 +20,6 @@ import org.apache.ignite.binary.BinaryRawReader; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache removeAll request. @@ -38,8 +37,6 @@ public ClientCacheRemoveAllRequest(BinaryRawReader reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_REMOVE); - cache(ctx).removeAll(); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java index 26c191f5b5553..b86f2f8895d64 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache remove request with value. @@ -39,8 +38,6 @@ public ClientCacheRemoveIfEqualsRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_REMOVE); - boolean res = cache(ctx).remove(key(), val()); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java index 5af9743b3cac3..a68c32730f4fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Remove request. @@ -39,8 +38,6 @@ public ClientCacheRemoveKeyRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_REMOVE); - boolean val = cache(ctx).remove(key()); return new ClientBooleanResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java index 62dea00201af8..043b5688a3f43 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java @@ -20,7 +20,6 @@ import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Remove keys request. @@ -38,8 +37,6 @@ public ClientCacheRemoveKeysRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_REMOVE); - cache(ctx).removeAll(keys()); return super.process(ctx); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java index 056367d71d2a9..8645fbb817322 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache replace request. @@ -44,8 +43,6 @@ public ClientCacheReplaceIfEqualsRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT); - boolean res = cache(ctx).replace(key(), val(), newVal); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java index ea04593e7cf15..bd7a642bb39e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse; import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; import org.apache.ignite.internal.processors.platform.client.ClientResponse; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache replace request. @@ -39,8 +38,6 @@ public ClientCacheReplaceRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT); - boolean res = cache(ctx).replace(key(), val()); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java index 9e39b5618c686..52b799f345120 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java @@ -24,8 +24,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientRequest; import org.apache.ignite.internal.processors.platform.client.ClientStatus; import org.apache.ignite.internal.processors.platform.client.IgniteClientException; -import org.apache.ignite.internal.processors.security.SecurityContext; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Cache get request. @@ -121,34 +119,4 @@ public static DynamicCacheDescriptor cacheDescriptor(ClientConnectionContext ctx protected int cacheId() { return cacheId; } - - /** {@inheritDoc} */ - @Override protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) { - SecurityContext secCtx = ctx.securityContext(); - - if (secCtx != null) { - DynamicCacheDescriptor cacheDesc = cacheDescriptor(ctx, cacheId); - - runWithSecurityExceptionHandler(() -> { - ctx.kernalContext().security().authorize(cacheDesc.cacheName(), perm, secCtx); - }); - } - } - - /** - * Authorize for multiple permissions. - */ - protected void authorize(ClientConnectionContext ctx, SecurityPermission... perm) - throws IgniteClientException { - SecurityContext secCtx = ctx.securityContext(); - - if (secCtx != null) { - DynamicCacheDescriptor cacheDesc = cacheDescriptor(ctx, cacheId); - - runWithSecurityExceptionHandler(() -> { - for (SecurityPermission p : perm) - ctx.kernalContext().security().authorize(cacheDesc.cacheName(), p, secCtx); - }); - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java index 70b6966e999c4..26ab236e8be1e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java @@ -28,7 +28,6 @@ import org.apache.ignite.internal.processors.platform.client.ClientResponse; import org.apache.ignite.internal.processors.platform.utils.PlatformUtils; import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.plugin.security.SecurityPermission; /** * Scan query request. @@ -81,8 +80,6 @@ public ClientCacheScanQueryRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { - authorize(ctx, SecurityPermission.CACHE_READ); - IgniteCache cache = filterPlatform == FILTER_PLATFORM_JAVA && !isKeepBinary() ? rawCache(ctx) : cache(ctx); ScanQuery qry = new ScanQuery() From 00f5a157396592c1acf459f90d4a5ec8a74deb86 Mon Sep 17 00:00:00 2001 From: Andrey Kuznetsov Date: Thu, 29 Nov 2018 14:54:43 +0300 Subject: [PATCH 188/403] IGNITE-10079 Fiexd WAL segments compression bug when FileWriteAheadLogManager return invalid lastCompactedSegment Signed-off-by: Andrey Gura (cherry picked from commit 2a09d54625c62acbb05ee71fb98c513b5e2c3183) --- .../wal/FileWriteAheadLogManager.java | 34 ++-- .../wal/aware/SegmentArchivedStorage.java | 16 ++ .../persistence/wal/aware/SegmentAware.java | 13 +- .../wal/aware/SegmentCompressStorage.java | 7 +- .../db/wal/WalCompactionAfterRestartTest.java | 161 ++++++++++++++++++ .../wal/aware/SegmentAwareTest.java | 10 +- 6 files changed, 211 insertions(+), 30 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionAfterRestartTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 3bc47e733d536..b428721305716 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -2067,7 +2067,13 @@ private long tryReserveNextSegmentOrWait() throws IgniteInterruptedCheckedExcept boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0)); - return reserved ? segmentToCompress : -1; + if (reserved) + return segmentToCompress; + else { + segmentAware.onSegmentCompressed(segmentToCompress); + + return -1; + } } /** {@inheritDoc} */ @@ -2081,9 +2087,7 @@ private void body0() { long segIdx = -1L; try { - segIdx = tryReserveNextSegmentOrWait(); - - if (segIdx <= segmentAware.lastCompressedIdx()) + if ((segIdx = tryReserveNextSegmentOrWait()) == -1) continue; deleteObsoleteRawSegments(); @@ -2102,21 +2106,19 @@ private void body0() { Files.move(tmpZip.toPath(), zip.toPath()); - if (mode != WALMode.NONE) { - try (FileIO f0 = ioFactory.create(zip, CREATE, READ, WRITE)) { - f0.force(); - } - - if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED) && !cctx.kernalContext().recoveryMode()) { - evt.record(new WalSegmentCompactedEvent( - cctx.localNode(), - segIdx, - zip.getAbsoluteFile()) - ); - } + try (FileIO f0 = ioFactory.create(zip, CREATE, READ, WRITE)) { + f0.force(); } segmentAware.onSegmentCompressed(segIdx); + + if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED) && !cctx.kernalContext().recoveryMode()) { + evt.record(new WalSegmentCompactedEvent( + cctx.localNode(), + segIdx, + zip.getAbsoluteFile()) + ); + } } catch (IgniteInterruptedCheckedException ignore) { Thread.currentThread().interrupt(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java index c526ae11e6feb..e31628f40abbf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java @@ -34,6 +34,8 @@ class SegmentArchivedStorage extends SegmentObservable { * no segments archived. */ private volatile long lastAbsArchivedIdx = -1; + /** Latest truncated segment. */ + private volatile long lastTruncatedArchiveIdx = -1; /** * @param segmentLockStorage Protects WAL work segments from moving. @@ -136,4 +138,18 @@ private void checkInterrupted() throws IgniteInterruptedCheckedException { private synchronized void onSegmentUnlocked(long segmentId) { notifyAll(); } + + /** + * @param lastTruncatedArchiveIdx Last truncated segment. + */ + void lastTruncatedArchiveIdx(long lastTruncatedArchiveIdx) { + this.lastTruncatedArchiveIdx = lastTruncatedArchiveIdx; + } + + /** + * @return Last truncated segment. + */ + long lastTruncatedArchiveIdx() { + return lastTruncatedArchiveIdx; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java index e46d93f015947..a14f0cac3a325 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java @@ -27,8 +27,6 @@ * Holder of actual information of latest manipulation on WAL segments. */ public class SegmentAware { - /** Latest truncated segment. */ - private volatile long lastTruncatedArchiveIdx = -1L; /** Segment reservations storage: Protects WAL segments from deletion during WAL log cleanup. */ private final SegmentReservationStorage reservationStorage = new SegmentReservationStorage(); /** Lock on segment protects from archiving segment. */ @@ -106,7 +104,12 @@ public void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedE * there's no segment to archive right now. */ public long waitNextSegmentToCompress() throws IgniteInterruptedCheckedException { - return Math.max(segmentCompressStorage.nextSegmentToCompressOrWait(), lastTruncatedArchiveIdx + 1); + long idx; + + while ((idx = segmentCompressStorage.nextSegmentToCompressOrWait()) <= lastTruncatedArchiveIdx()) + onSegmentCompressed(idx); + + return idx; } /** @@ -152,14 +155,14 @@ public void curAbsWalIdx(long curAbsWalIdx) { * @param lastTruncatedArchiveIdx Last truncated segment; */ public void lastTruncatedArchiveIdx(long lastTruncatedArchiveIdx) { - this.lastTruncatedArchiveIdx = lastTruncatedArchiveIdx; + segmentArchivedStorage.lastTruncatedArchiveIdx(lastTruncatedArchiveIdx); } /** * @return Last truncated segment. */ public long lastTruncatedArchiveIdx() { - return lastTruncatedArchiveIdx; + return segmentArchivedStorage.lastTruncatedArchiveIdx(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java index 174fb46bb6070..95d4f4a770c39 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java @@ -94,6 +94,9 @@ synchronized void onSegmentCompressed(long compressedIdx) { this.lastCompressedIdx = Math.min(lastMaxCompressedIdx, compressingSegments.get(0) - 1); else this.lastCompressedIdx = lastMaxCompressedIdx; + + if (compressedIdx > lastEnqueuedToCompressIdx) + lastEnqueuedToCompressIdx = compressedIdx; } /** @@ -120,9 +123,11 @@ synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedE Long idx = segmentsToCompress.poll(); + assert idx != null; + compressingSegments.add(idx); - return idx == null ? -1L : idx; + return idx; } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionAfterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionAfterRestartTest.java new file mode 100644 index 0000000000000..3685fe7635683 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionAfterRestartTest.java @@ -0,0 +1,161 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ignite.internal.processors.cache.persistence.db.wal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.events.WalSegmentCompactedEvent; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_COMPACTED; + +/** */ +public class WalCompactionAfterRestartTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String name) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(name); + + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(200L * 1024 * 1024)) + .setWalMode(WALMode.LOG_ONLY) + .setWalSegmentSize(512 * 1024) + .setWalCompactionEnabled(true) + .setMaxWalArchiveSize(2 * 512 * 1024) + ); + + CacheConfiguration ccfg = new CacheConfiguration(); + + ccfg.setName(DEFAULT_CACHE_NAME); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + ccfg.setAffinity(new RendezvousAffinityFunction(false, 16)); + ccfg.setBackups(0); + + cfg.setCacheConfiguration(ccfg); + cfg.setConsistentId(name); + + cfg.setIncludeEventTypes(EVT_WAL_SEGMENT_COMPACTED); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** + * @throws Exception If failed. + */ + public void test() throws Exception { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + doCachePuts(ig, 10_000); + + ig.cluster().active(false); + + stopGrid(0); + + IgniteEx ig0 = startGrid(0); + + ig0.cluster().active(true); + + List> discrepancies = Collections.synchronizedList(new ArrayList<>()); + + ig0.events().localListen(e -> { + long evtSegIdx = ((WalSegmentCompactedEvent)e).getAbsWalSegmentIdx(); + long lastCompactedIdx = ig0.context().cache().context().wal().lastCompactedSegment(); + + if (lastCompactedIdx < 0 || lastCompactedIdx > evtSegIdx) + discrepancies.add(F.t(evtSegIdx, lastCompactedIdx)); + + return true; + }, EVT_WAL_SEGMENT_COMPACTED); + + doCachePuts(ig0, 5_000); + + stopGrid(0); + + if (!discrepancies.isEmpty()) { + fail("Discrepancies (EVT_WAL_SEGMENT_COMPACTED index vs. lastCompactedSegment):" + System.lineSeparator() + + discrepancies.stream() + .map(t -> String.format("%d <-> %d", t.get1(), t.get2())) + .collect(Collectors.joining(System.lineSeparator()))); + } + } + + /** */ + private void doCachePuts(IgniteEx ig, long millis) throws IgniteCheckedException { + IgniteCache cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME); + + AtomicBoolean stop = new AtomicBoolean(); + + IgniteInternalFuture putFut = GridTestUtils.runMultiThreadedAsync(() -> { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + while (!stop.get()) + cache.put(rnd.nextInt(), "Ignite".getBytes()); + }, 4, "cache-filler"); + + U.sleep(millis); + + stop.set(true); + + putFut.get(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java index 22b30f140e552..923e05204ca26 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java @@ -437,18 +437,12 @@ public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws * Next segment for compress based on truncated archive idx. */ public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedException, InterruptedException { - //given: thread which awaited segment. SegmentAware aware = new SegmentAware(10, true); - aware.onSegmentCompressed(5); aware.setLastArchivedAbsoluteIndex(6); - aware.lastTruncatedArchiveIdx(7); - - //when: - long segmentToCompress = aware.waitNextSegmentToCompress(); - //then: segment to compress greater than truncated archive idx - assertEquals(8, segmentToCompress); + for (int exp = 0; exp <= 6; exp++) + assertEquals(exp, aware.waitNextSegmentToCompress()); } /** From b6afd0d9cee8b1203c98c0c841993761bbe29353 Mon Sep 17 00:00:00 2001 From: Roman Guseinov Date: Thu, 29 Nov 2018 16:40:46 +0300 Subject: [PATCH 189/403] IGNITE-10354 Failing client node due to not receiving metrics updates - Fixes #5485. Signed-off-by: Ilya Kasnacheev (cherry picked from commit d4496c7) --- .../ignite/spi/discovery/tcp/ServerImpl.java | 3 + .../tcp/internal/TcpDiscoveryNode.java | 9 +- ...ientDiscoverySpiCoordinatorChangeTest.java | 121 ++++++++++++++++++ .../IgniteSpiDiscoverySelfTestSuite.java | 2 + 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiCoordinatorChangeTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 15917f4c5e454..e6fd6d62e575a 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -5274,6 +5274,9 @@ private void processMetricsUpdateMessage(TcpDiscoveryMetricsUpdateMessage msg) { if (clientNodeIds.contains(clientNode.id())) clientNode.clientAliveTime(spi.clientFailureDetectionTimeout()); else { + if (clientNode.clientAliveTime() == 0L) + clientNode.clientAliveTime(spi.clientFailureDetectionTimeout()); + boolean aliveCheck = clientNode.isClientAlive(); if (!aliveCheck && isLocalNodeCoordinator()) { diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java index 13d100652d3c0..763a678b9ade1 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java @@ -124,7 +124,7 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Ignite /** Alive check time (used by clients). */ @GridToStringExclude - private transient long aliveCheckTime; + private transient volatile long aliveCheckTime; /** Client router node ID. */ @GridToStringExclude @@ -499,6 +499,13 @@ public void clientAliveTime(long aliveTime) { this.aliveCheckTime = U.currentTimeMillis() + aliveTime; } + /** + * @return Client alive check time. + */ + public long clientAliveTime() { + return aliveCheckTime; + } + /** * @return Client router node ID. */ diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiCoordinatorChangeTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiCoordinatorChangeTest.java new file mode 100644 index 0000000000000..0939e6323ca4b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiCoordinatorChangeTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.tcp; + +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.Event; +import org.apache.ignite.events.EventType; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * This class tests that a client is able to connect to another server node without leaving the cluster. + */ +public class TcpClientDiscoverySpiCoordinatorChangeTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * Checks that a client node doesn't fail because of coordinator change. + * + * @throws Exception If test fails. + */ + public void testClientNotFailed() throws Exception { + TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + // Start server A. + Ignite srvA = startNode("server-a", ipFinder, false); + + // Start the client. + Ignite client = startNode("client", ipFinder, true); + + AtomicBoolean clientReconnectState = getClientReconnectState(client); + + // Start server B. + Ignite srvB = startNode("server-b", ipFinder, false); + + // Stop server A. + srvA.close(); + + // Will throw an exception if the client is disconnected. + client.getOrCreateCache("CACHE-NAME"); + + // Check that the client didn't disconnect/reconnect quickly. + assertFalse("Client node was failed and reconnected to the cluster.", clientReconnectState.get()); + + // Stop the client. + client.close(); + + // Stop server B. + srvB.close(); + } + + /** + * @param instanceName Instance name. + * @param ipFinder IP-finder. + * @param clientMode Client mode flag. + * @return Started node. + * @throws Exception If a node was not started. + */ + private Ignite startNode(String instanceName, TcpDiscoveryIpFinder ipFinder, boolean clientMode) throws Exception { + IgniteConfiguration cfg = getConfiguration(instanceName) + .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(ipFinder)) + .setClientMode(clientMode); + + return Ignition.start(cfg); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception { + return super.getConfiguration(instanceName) + .setMetricsUpdateFrequency(Integer.MAX_VALUE) + .setClientFailureDetectionTimeout(Integer.MAX_VALUE) + .setFailureDetectionTimeout(Integer.MAX_VALUE); + } + + /** + * @param ignite Client node. + * @return Client reconnect state. + */ + private AtomicBoolean getClientReconnectState(Ignite ignite) { + final AtomicBoolean reconnectState = new AtomicBoolean(false); + + ignite.events().localListen( + new IgnitePredicate() { + @Override public boolean apply(Event evt) { + if (evt.type() == EventType.EVT_CLIENT_NODE_RECONNECTED) + reconnectState.set(true); + + return true; + } + }, + EventType.EVT_CLIENT_NODE_RECONNECTED + ); + + return reconnectState; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java index 80f093dfd68c2..3ef1087a81efd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java @@ -28,6 +28,7 @@ import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest; import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest; +import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiCoordinatorChangeTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest; @@ -100,6 +101,7 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(TcpClientDiscoverySpiSelfTest.class)); suite.addTest(new TestSuite(LongClientConnectToClusterTest.class)); suite.addTest(new TestSuite(TcpClientDiscoveryMarshallerCheckSelfTest.class)); + suite.addTest(new TestSuite(TcpClientDiscoverySpiCoordinatorChangeTest.class)); suite.addTest(new TestSuite(TcpClientDiscoverySpiMulticastTest.class)); suite.addTest(new TestSuite(TcpClientDiscoverySpiFailureTimeoutSelfTest.class)); suite.addTest(new TestSuite(TcpClientDiscoveryUnresolvedHostTest.class)); From 66eefef1ce50921827f1b0d09283b51d27ef4517 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Fri, 30 Nov 2018 17:38:53 +0700 Subject: [PATCH 190/403] IGNITE-10490 Web Console: Fixed ui-grid refresh on its parent element resize. (cherry picked from commit 19772109d83c0caa348b9522135ea1f2e8463354) --- .../frontend/app/components/ui-grid/controller.js | 10 ++++++++++ .../frontend/app/components/ui-grid/style.scss | 1 + modules/web-console/frontend/app/vendor.js | 2 ++ modules/web-console/frontend/package.json | 3 ++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/ui-grid/controller.js b/modules/web-console/frontend/app/components/ui-grid/controller.js index f53416b57e7f0..ec7101b0b3a6c 100644 --- a/modules/web-console/frontend/app/components/ui-grid/controller.js +++ b/modules/web-console/frontend/app/components/ui-grid/controller.js @@ -18,6 +18,8 @@ import debounce from 'lodash/debounce'; import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug'; +import ResizeObserver from 'resize-observer-polyfill'; + export default class IgniteUiGrid { /** @type {import('ui-grid').IGridOptions} */ grid; @@ -129,6 +131,9 @@ export default class IgniteUiGrid { this.$timeout(() => { if (this.selectedRowsId) this.applyIncomingSelectionRowsId(this.selectedRowsId); }); + + this.resizeObserver = new ResizeObserver(() => api.core.handleWindowResize()); + this.resizeObserver.observe(this.$element[0]); } }; @@ -153,6 +158,11 @@ export default class IgniteUiGrid { this.adjustHeight(); } + $onDestroy() { + if (this.resizeObserver) + this.resizeObserver.disconnect(); + } + applyIncomingSelectionRows = (selected = []) => { this.gridApi.selection.clearSelectedRows({ ignore: true }); diff --git a/modules/web-console/frontend/app/components/ui-grid/style.scss b/modules/web-console/frontend/app/components/ui-grid/style.scss index dba5d359ea290..da31f67efab05 100644 --- a/modules/web-console/frontend/app/components/ui-grid/style.scss +++ b/modules/web-console/frontend/app/components/ui-grid/style.scss @@ -18,6 +18,7 @@ .ignite-grid-table, ignite-grid-table { @import 'public/stylesheets/variables'; + display: block; .ui-grid.ui-grid--ignite.ui-grid--thin { // Start section row height. diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js index 6961e962e30b2..90808be43ad55 100644 --- a/modules/web-console/frontend/app/vendor.js +++ b/modules/web-console/frontend/app/vendor.js @@ -28,6 +28,8 @@ import 'angular-messages'; import '@uirouter/angularjs'; import '@uirouter/angularjs/lib/legacy/stateEvents'; +import 'resize-observer-polyfill'; + import 'tf-metatags'; import 'angular-translate'; import 'angular-smart-table'; diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json index 9059a5f3e2fd8..e8f4ae2f05320 100644 --- a/modules/web-console/frontend/package.json +++ b/modules/web-console/frontend/package.json @@ -68,6 +68,7 @@ "nvd3": "1.8.6", "outdent": "0.5.0", "pako": "1.0.6", + "resize-observer-polyfill": "1.5.0", "roboto-font": "0.1.0", "rxjs": "5.5.11", "socket.io-client": "1.7.3", @@ -129,7 +130,7 @@ "mocha": "3.4.2", "mocha-teamcity-reporter": "1.1.1", "node-fetch": "1.7.3", - "node-sass": "4.9.3", + "node-sass": "4.10.0", "progress": "2.0.0", "progress-bar-webpack-plugin": "1.11.0", "pug-html-loader": "1.1.0", From 55b4e6913c92c46fe60aaf7190537e9174940412 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Fri, 30 Nov 2018 14:58:45 +0300 Subject: [PATCH 191/403] IGNITE-10236: MVCC: Create "Cache 3" test suite for MVCC mode. This closes #5445. (cherry picked from commit 00e4dc0b13b86bfb72fa1b888554b19660685d44) --- ...ceptorPartitionCounterLocalSanityTest.java | 23 ++ ...rPartitionCounterRandomOperationsTest.java | 104 +++++++- ...tomicEntryProcessorDeploymentSelfTest.java | 11 + .../cache/GridCacheEntryVersionSelfTest.java | 10 + .../GridCacheInterceptorAbstractSelfTest.java | 9 + .../GridCacheInterceptorLocalSelfTest.java | 8 + ...acheInterceptorLocalWithStoreSelfTest.java | 9 + ...InterceptorTransactionalRebalanceTest.java | 8 + .../cache/GridCachePartitionedWritesTest.java | 8 + .../GridCacheReferenceCleanupSelfTest.java | 17 ++ .../cache/GridCacheReloadSelfTest.java | 14 ++ ...CacheValueConsistencyAbstractSelfTest.java | 6 + .../GridCacheVersionTopologyChangeTest.java | 8 + .../cache/IgniteCacheGroupsTest.java | 228 +++++++++++++++++- .../IgniteCacheInterceptorSelfTestSuite.java | 57 +++-- .../distributed/CacheAsyncOperationsTest.java | 8 + ...actDistributedByteArrayValuesSelfTest.java | 4 + ...actPartitionedByteArrayValuesSelfTest.java | 10 + .../GridCacheClientModesAbstractSelfTest.java | 4 + .../GridCacheEventAbstractTest.java | 3 + .../GridCacheMixedModeSelfTest.java | 2 + .../GridCacheMultiNodeAbstractTest.java | 2 + ...ridCachePreloadEventsAbstractSelfTest.java | 8 + .../IgniteTxPreloadAbstractTest.java | 2 +- .../IgniteTxRemoveTimeoutObjectsTest.java | 12 + ...artitionedOnlyByteArrayValuesSelfTest.java | 3 - ...AbstractTransformWriteThroughSelfTest.java | 3 + .../GridCacheNearOnlyTopologySelfTest.java | 8 + .../GridCachePartitionedEventSelfTest.java | 8 - .../near/GridCachePutArrayValueSelfTest.java | 9 + ...ReplicatedMvccTxMultiThreadedSelfTest.java | 81 +++++++ ...eplicatedMvccTxSingleThreadedSelfTest.java | 77 ++++++ ...dCacheReplicatedMvccTxTimeoutSelfTest.java | 44 ++++ ...acheReplicatedTxMultiThreadedSelfTest.java | 25 -- ...cheReplicatedTxSingleThreadedSelfTest.java | 44 +--- .../GridCacheReplicatedPreloadSelfTest.java | 134 +++------- .../GridCacheDaemonNodeLocalSelfTest.java | 8 + ...GridCacheLocalByteArrayValuesSelfTest.java | 3 + .../local/GridCacheLocalEventSelfTest.java | 2 +- ...GridCacheWriteBehindStoreAbstractTest.java | 8 + .../GridCacheWriteBehindStoreLocalTest.java | 8 + ...hindStorePartitionedMultiNodeSelfTest.java | 8 + ...cheClientWriteBehindStoreAbstractTest.java | 8 + .../testframework/MvccFeatureChecker.java | 6 +- .../IgniteBinaryObjectsCacheTestSuite3.java | 18 +- .../testsuites/IgniteCacheMvccTestSuite3.java | 133 ++++++++++ .../IgniteCacheNearOnlySelfTestSuite.java | 31 ++- .../testsuites/IgniteCacheTestSuite2.java | 6 +- .../testsuites/IgniteCacheTestSuite3.java | 161 +++++++------ .../IgniteCacheWriteBehindTestSuite.java | 33 ++- 50 files changed, 1129 insertions(+), 315 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxMultiThreadedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxSingleThreadedSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxTimeoutSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite3.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterLocalSanityTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterLocalSanityTest.java index acb929b5672ef..3be51e93da7e0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterLocalSanityTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterLocalSanityTest.java @@ -52,6 +52,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; @@ -137,6 +138,28 @@ public void testLocalTxWithStore() throws Exception { doTestPartitionCounterOperation(ccfg); } + /** + * @throws Exception If failed. + */ + public void testLocalMvccTx() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + CacheConfiguration ccfg = cacheConfiguration(TRANSACTIONAL_SNAPSHOT,false); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testLocalMvccTxWithStore() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + CacheConfiguration ccfg = cacheConfiguration(TRANSACTIONAL_SNAPSHOT,true); + + doTestPartitionCounterOperation(ccfg); + } + /** * @param ccfg Cache configuration. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterRandomOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterRandomOperationsTest.java index 74170a6559a65..385091a91473d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterRandomOperationsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheInterceptorPartitionCounterRandomOperationsTest.java @@ -62,6 +62,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -290,12 +291,111 @@ public void testTxNoBackupsExplicit() throws Exception { doTestPartitionCounterOperation(ccfg); } + /** + * @throws Exception If failed. + */ + public void testMvccTx() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, + 1, + TRANSACTIONAL_SNAPSHOT, + false); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxWithStore() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, + 1, + TRANSACTIONAL_SNAPSHOT, + true); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxExplicit() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, + 1, + TRANSACTIONAL_SNAPSHOT, + false); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxReplicated() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(REPLICATED, + 0, + TRANSACTIONAL_SNAPSHOT, + false); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxReplicatedWithStore() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(REPLICATED, + 0, + TRANSACTIONAL_SNAPSHOT, + true); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxNoBackups() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, + 0, + TRANSACTIONAL_SNAPSHOT, + false); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxNoBackupsWithStore() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, + 0, + TRANSACTIONAL_SNAPSHOT, + true); + + doTestPartitionCounterOperation(ccfg); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxNoBackupsExplicit() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PARTITIONED, + 0, + TRANSACTIONAL_SNAPSHOT, + false); + + doTestPartitionCounterOperation(ccfg); + } + /** * @param ccfg Cache configuration. * @throws Exception If failed. */ protected void doTestPartitionCounterOperation(CacheConfiguration ccfg) throws Exception { + if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + fail("https://issues.apache.org/jira/browse/IGNITE-9323"); + ignite(0).createCache(ccfg); try { @@ -345,7 +445,9 @@ private void randomUpdate( Transaction tx = null; - if (cache.getConfiguration(CacheConfiguration.class).getAtomicityMode() == TRANSACTIONAL && rnd.nextBoolean()) + CacheAtomicityMode atomicityMode = cache.getConfiguration(CacheConfiguration.class).getAtomicityMode(); + + if (atomicityMode == TRANSACTIONAL && rnd.nextBoolean()) tx = ignite.transactions().txStart(txRandomConcurrency(rnd), txRandomIsolation(rnd)); try { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAtomicEntryProcessorDeploymentSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAtomicEntryProcessorDeploymentSelfTest.java index 4f513b3845668..f616b6e1605ed 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAtomicEntryProcessorDeploymentSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAtomicEntryProcessorDeploymentSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.config.GridTestProperties; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -54,6 +55,16 @@ public class GridCacheAtomicEntryProcessorDeploymentSelfTest extends GridCommonA /** */ protected boolean clientMode; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + assert atomicityMode() != ATOMIC; + + fail("https://issues.apache.org/jira/browse/IGNITE-10359"); + } + + super.beforeTestsStarted(); + } /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryVersionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryVersionSelfTest.java index 9e933be2acfca..8a4196b84837e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryVersionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryVersionSelfTest.java @@ -31,6 +31,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager.TOP_VER_BASE_TIME; @@ -85,6 +86,15 @@ public void testVersionTransactional() throws Exception { checkVersion(); } + /** + * @throws Exception If failed. + */ + public void testVersionMvccTx() throws Exception { + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + checkVersion(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorAbstractSelfTest.java index 432011e2c549b..12e8042292a23 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorAbstractSelfTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -68,6 +69,14 @@ public abstract class GridCacheInterceptorAbstractSelfTest extends GridCacheAbst /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.INTERCEPTOR); + + if (nearEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + if (storeEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + interceptor = new Interceptor(); super.beforeTestsStarted(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalSelfTest.java index 3ee9383d0c72a..b8573fc4ce33c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -27,6 +28,13 @@ * Tests {@link org.apache.ignite.cache.CacheInterceptor}. */ public class GridCacheInterceptorLocalSelfTest extends GridCacheInterceptorAbstractSelfTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @SuppressWarnings("RedundantMethodOverride") @Override protected CacheAtomicityMode atomicityMode() { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalWithStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalWithStoreSelfTest.java index 15679b0ad9951..6375aa66ea2d0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalWithStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorLocalWithStoreSelfTest.java @@ -17,10 +17,19 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.testframework.MvccFeatureChecker; + /** * Tests {@link org.apache.ignite.cache.CacheInterceptor}. */ public class GridCacheInterceptorLocalWithStoreSelfTest extends GridCacheInterceptorLocalSelfTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected boolean storeEnabled() { return true; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorTransactionalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorTransactionalRebalanceTest.java index 87c3f32e7ce57..d82b14d14fd2e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorTransactionalRebalanceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheInterceptorTransactionalRebalanceTest.java @@ -18,11 +18,19 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * */ public class GridCacheInterceptorTransactionalRebalanceTest extends GridAbstractCacheInterceptorRebalanceTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.INTERCEPTOR); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return CacheAtomicityMode.TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedWritesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedWritesTest.java index 0d00bef472dac..a0c16699ffdda 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedWritesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePartitionedWritesTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -42,6 +43,13 @@ public class GridCachePartitionedWritesTest extends GridCommonAbstractTest { /** Cache store. */ private CacheStore store; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected final IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReferenceCleanupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReferenceCleanupSelfTest.java index e6a40a64c1819..77e080082fa02 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReferenceCleanupSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReferenceCleanupSelfTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -80,6 +81,9 @@ public class GridCacheReferenceCleanupSelfTest extends GridCommonAbstractTest { /** @throws Exception If failed. */ public void testAtomicLongPartitioned() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + mode = CacheMode.PARTITIONED; startGrids(2); @@ -94,6 +98,9 @@ public void testAtomicLongPartitioned() throws Exception { /** @throws Exception If failed. */ public void testAtomicLongReplicated() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + mode = CacheMode.REPLICATED; startGrids(2); @@ -108,6 +115,8 @@ public void testAtomicLongReplicated() throws Exception { /** @throws Exception If failed. */ public void testAtomicLongLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = CacheMode.LOCAL; try { @@ -148,6 +157,8 @@ public void testOneAsyncOpReplicated() throws Exception { /** @throws Exception If failed. */ public void testOneAsyncOpLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = CacheMode.LOCAL; try { @@ -188,6 +199,8 @@ public void testSeveralAsyncOpsReplicated() throws Exception { /** @throws Exception If failed. */ public void testSeveralAsyncOpsLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = CacheMode.LOCAL; try { @@ -228,6 +241,8 @@ public void testSyncOpAsyncCommitReplicated() throws Exception { /** @throws Exception If failed. */ public void testSyncOpAsyncCommitLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = CacheMode.LOCAL; try { @@ -268,6 +283,8 @@ public void testAsyncOpsAsyncCommitReplicated() throws Exception { /** @throws Exception If failed. */ public void testAsyncOpsAsyncCommitLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = CacheMode.LOCAL; try { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReloadSelfTest.java index 50bed2a611f99..b9603837f02b5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReloadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheReloadSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -53,6 +54,17 @@ public class GridCacheReloadSelfTest extends GridCommonAbstractTest { /** Near enabled flag. */ private boolean nearEnabled = true; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + + if (nearEnabled) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { cacheMode = null; @@ -119,6 +131,8 @@ public class GridCacheReloadSelfTest extends GridCommonAbstractTest { * @throws Exception If error occurs. */ public void testReloadEvictionLocalCache() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + cacheMode = CacheMode.LOCAL; doTest(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java index 19f98ff8ab373..c5059863af8de 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java @@ -26,6 +26,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -68,6 +69,11 @@ public abstract class GridCacheValueConsistencyAbstractSelfTest extends GridCach /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + if (nearEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + // Need to increase value set in GridAbstractTest sizePropVal = System.getProperty(IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionTopologyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionTopologyChangeTest.java index 5c9ecd71f6d13..8b70f734e808d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionTopologyChangeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionTopologyChangeTest.java @@ -39,6 +39,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** @@ -78,6 +79,13 @@ public void testVersionIncreaseTx() throws Exception { checkVersionIncrease(cacheConfigurations(TRANSACTIONAL)); } + /** + * @throws Exception If failed. + */ + public void testVersionIncreaseMvccTx() throws Exception { + checkVersionIncrease(cacheConfigurations(TRANSACTIONAL_SNAPSHOT)); + } + /** * @param ccfgs Cache configurations. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java index 2104c5d91ecb4..022674f1ab63c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache; +import com.google.common.collect.Sets; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -49,7 +50,6 @@ import javax.cache.integration.CacheWriterException; import javax.cache.processor.EntryProcessorException; import javax.cache.processor.MutableEntry; -import com.google.common.collect.Sets; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; @@ -104,6 +104,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -127,6 +128,9 @@ public class IgniteCacheGroupsTest extends GridCommonAbstractTest { /** */ private static final String GROUP2 = "grp2"; + /** */ + private static final String GROUP3 = "grp3"; + /** */ private static final String CACHE1 = "cache1"; @@ -251,6 +255,13 @@ public void testCreateDestroyCachesTxPartitioned() throws Exception { createDestroyCaches(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCreateDestroyCachesMvccTxPartitioned() throws Exception { + createDestroyCaches(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -265,6 +276,13 @@ public void testCreateDestroyCachesTxReplicated() throws Exception { createDestroyCaches(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCreateDestroyCachesMvccTxReplicated() throws Exception { + createDestroyCaches(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -279,6 +297,13 @@ public void testScanQueryTxPartitioned() throws Exception { scanQuery(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testScanQueryMvccTxPartitioned() throws Exception { + scanQuery(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -293,6 +318,14 @@ public void testScanQueryTxReplicated() throws Exception { scanQuery(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testScanQueryMvccTxReplicated() throws Exception { + scanQuery(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + + /** * @throws Exception If failed. */ @@ -307,6 +340,15 @@ public void testScanQueryTxLocal() throws Exception { scanQuery(LOCAL, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testScanQueryMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + scanQuery(LOCAL, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -321,6 +363,15 @@ public void testEntriesTtlTxPartitioned() throws Exception { entriesTtl(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testEntriesTtlMvccTxPartitioned() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + + entriesTtl(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -335,6 +386,15 @@ public void testEntriesTtlTxReplicated() throws Exception { entriesTtl(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testEntriesTtlMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + + entriesTtl(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -349,6 +409,16 @@ public void testEntriesTtlTxLocal() throws Exception { entriesTtl(LOCAL, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testEntriesTtlMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + + entriesTtl(LOCAL, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -363,6 +433,13 @@ public void testCacheIteratorTxPartitioned() throws Exception { cacheIterator(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCacheIteratorMvccTxPartitioned() throws Exception { + cacheIterator(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -377,6 +454,13 @@ public void testCacheIteratorTxReplicated() throws Exception { cacheIterator(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCacheIteratorMvccTxReplicated() throws Exception { + cacheIterator(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -391,6 +475,15 @@ public void testCacheIteratorTxLocal() throws Exception { cacheIterator(LOCAL, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCacheIteratorMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + cacheIterator(LOCAL, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -405,6 +498,13 @@ public void testScanQueryMultiplePartitionsTxPartitioned() throws Exception { scanQueryMultiplePartitions(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testScanQueryMultiplePartitionsMvccTxPartitioned() throws Exception { + scanQueryMultiplePartitions(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -419,6 +519,13 @@ public void testScanQueryMultiplePartitionsTxReplicated() throws Exception { scanQueryMultiplePartitions(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testScanQueryMultiplePartitionsMvccTxReplicated() throws Exception { + scanQueryMultiplePartitions(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -426,6 +533,13 @@ public void testContinuousQueryTxReplicated() throws Exception { continuousQuery(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testContinuousQueryMvccTxReplicated() throws Exception { + continuousQuery(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -433,6 +547,13 @@ public void testContinuousQueryTxPartitioned() throws Exception { continuousQuery(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testContinuousQueryMvccTxPartitioned() throws Exception { + continuousQuery(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -440,6 +561,15 @@ public void testContinuousQueryTxLocal() throws Exception { continuousQuery(LOCAL, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testContinuousQueryMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + continuousQuery(LOCAL, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -1341,6 +1471,10 @@ public void testRebalance1() throws Exception { srv0.createCache(cacheConfiguration(GROUP2, "c3", PARTITIONED, TRANSACTIONAL, 2, false)); IgniteCache srv0Cache4 = srv0.createCache(cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 2, false)); + IgniteCache srv0Cache5 = + srv0.createCache(cacheConfiguration(GROUP3, "c5", PARTITIONED, TRANSACTIONAL_SNAPSHOT, 2, false)); + IgniteCache srv0Cache6 = + srv0.createCache(cacheConfiguration(GROUP3, "c6", PARTITIONED, TRANSACTIONAL_SNAPSHOT, 2, false)); final int ITEMS = 1_000; @@ -1349,6 +1483,9 @@ public void testRebalance1() throws Exception { srv0Cache3.put(new Key1(i), i); srv0Cache4.put(new Key1(i), -i); + + srv0Cache5.put(new Key1(i), i); + srv0Cache6.put(new Key1(i), -i); } assertEquals(ITEMS, srv0Cache1.size()); @@ -1356,6 +1493,8 @@ public void testRebalance1() throws Exception { assertEquals(0, srv0Cache2.size()); assertEquals(ITEMS, srv0Cache3.size()); assertEquals(ITEMS, srv0Cache4.localSize()); + assertEquals(ITEMS, srv0Cache5.size()); + assertEquals(ITEMS, srv0Cache6.localSize()); startGrid(1); @@ -1368,6 +1507,8 @@ public void testRebalance1() throws Exception { IgniteCache cache2 = node.cache("c2"); IgniteCache cache3 = node.cache("c3"); IgniteCache cache4 = node.cache("c4"); + IgniteCache cache5 = node.cache("c5"); + IgniteCache cache6 = node.cache("c6"); assertEquals(ITEMS * 2, cache1.size(CachePeekMode.ALL)); assertEquals(ITEMS, cache1.localSize(CachePeekMode.ALL)); @@ -1380,11 +1521,20 @@ public void testRebalance1() throws Exception { assertEquals(ITEMS * 2, cache4.size(CachePeekMode.ALL)); assertEquals(ITEMS, cache4.localSize(CachePeekMode.ALL)); + assertEquals(ITEMS * 2, cache5.size(CachePeekMode.ALL)); + assertEquals(ITEMS, cache5.localSize(CachePeekMode.ALL)); + + assertEquals(ITEMS * 2, cache6.size(CachePeekMode.ALL)); + assertEquals(ITEMS, cache6.localSize(CachePeekMode.ALL)); + + for (int k = 0; k < ITEMS; k++) { assertEquals(i, cache1.localPeek(new Key1(i))); assertNull(cache2.localPeek(new Key1(i))); assertEquals(i, cache3.localPeek(new Key1(i))); assertEquals(-i, cache4.localPeek(new Key1(i))); + assertEquals(i, cache5.localPeek(new Key1(i))); + assertEquals(-i, cache6.localPeek(new Key1(i))); } } @@ -1402,6 +1552,8 @@ public void testRebalance1() throws Exception { IgniteCache cache2 = node.cache("c2"); IgniteCache cache3 = node.cache("c3"); IgniteCache cache4 = node.cache("c4"); + IgniteCache cache5 = node.cache("c5"); + IgniteCache cache6 = node.cache("c6"); assertEquals(ITEMS * 3, cache1.size(CachePeekMode.ALL)); assertEquals(ITEMS, cache1.localSize(CachePeekMode.ALL)); @@ -1409,6 +1561,8 @@ public void testRebalance1() throws Exception { assertEquals(ITEMS * 2, cache2.localSize(CachePeekMode.ALL)); assertEquals(ITEMS, cache3.localSize(CachePeekMode.ALL)); assertEquals(ITEMS, cache4.localSize(CachePeekMode.ALL)); + assertEquals(ITEMS, cache5.localSize(CachePeekMode.ALL)); + assertEquals(ITEMS, cache6.localSize(CachePeekMode.ALL)); } IgniteCache srv2Cache1 = srv2.cache("c1"); @@ -1511,6 +1665,13 @@ public void testNoKeyIntersectTx() throws Exception { testNoKeyIntersect(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testNoKeyIntersectMvccTx() throws Exception { + testNoKeyIntersect(TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -1829,6 +1990,15 @@ public void testCacheApiTxPartitioned() throws Exception { cacheApiTest(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCacheApiMvccTxPartitioned() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + + cacheApiTest(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -1836,6 +2006,15 @@ public void testCacheApiTxReplicated() throws Exception { cacheApiTest(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCacheApiMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + + cacheApiTest(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -2656,6 +2835,15 @@ public void testLoadCacheTxPartitioned() throws Exception { loadCache(PARTITIONED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testLoadCacheMvccTxPartitioned() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7954"); + + loadCache(PARTITIONED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -2663,6 +2851,15 @@ public void testLoadCacheTxReplicated() throws Exception { loadCache(REPLICATED, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testLoadCacheMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7954"); + + loadCache(REPLICATED, TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -2677,6 +2874,15 @@ public void testLoadCacheTxLocal() throws Exception { loadCache(LOCAL, TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testLoadCacheMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + loadCache(LOCAL, TRANSACTIONAL_SNAPSHOT); + } + /** * @param cacheMode Cache mode. * @param atomicityMode Atomicity mode. @@ -2854,6 +3060,8 @@ public void testConcurrentOperationsAndCacheDestroy() throws Exception { srv0.createCache( cacheConfiguration(GROUP2, GROUP2 + "-" + i, PARTITIONED, TRANSACTIONAL, grp2Backups, i % 2 == 0)); + + // TODO IGNITE-7164: add Mvcc cache to test. } final AtomicInteger idx = new AtomicInteger(); @@ -3305,6 +3513,16 @@ public void testConfigurationConsistencyValidation() throws Exception { assertTrue("Unexpected message: " + e.getMessage(), e.getMessage().contains("Backups mismatch for caches related to the same group [groupName=grp1")); } + + try { + ignite(i).createCache(cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, false)); + + fail(); + } + catch (CacheException e) { + assertTrue("Unexpected message: " + e.getMessage(), + e.getMessage().contains("Atomicity mode mismatch for caches related to the same group [groupName=grp1")); + } } } @@ -3321,6 +3539,8 @@ private CacheConfiguration[] interceptorConfigurations() { ccfgs[4] = cacheConfiguration(GROUP1, "c5", PARTITIONED, ATOMIC, 2, false); ccfgs[5] = cacheConfiguration(GROUP1, "c6", PARTITIONED, TRANSACTIONAL, 2, false); + //TODO IGNITE-9323: Check Mvcc mode. + return ccfgs; } @@ -3384,6 +3604,8 @@ private CacheConfiguration[] cacheStoreConfigurations() { ccfgs[4] = cacheConfiguration(GROUP1, "c5", PARTITIONED, ATOMIC, 2, false); ccfgs[5] = cacheConfiguration(GROUP1, "c6", PARTITIONED, TRANSACTIONAL, 2, false); + //TODO IGNITE-8582: Check Mvcc mode. + return ccfgs; } @@ -3559,7 +3781,9 @@ private void continuousQueriesMultipleGroups(int srvs) throws Exception { client.createCache(cacheConfiguration(null, "c7", PARTITIONED, ATOMIC, 1, false)); client.createCache(cacheConfiguration(null, "c8", PARTITIONED, TRANSACTIONAL, 1, false)); - String[] cacheNames = {"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8"}; + client.createCache(cacheConfiguration(GROUP3, "c9", PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, false)); + + String[] cacheNames = {"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9"}; AtomicInteger c1 = registerListener(client, "c1"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInterceptorSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInterceptorSelfTestSuite.java index f457e5df8d4db..37b8d400d645d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInterceptorSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInterceptorSelfTestSuite.java @@ -17,7 +17,9 @@ package org.apache.ignite.internal.processors.cache; +import java.util.Collection; import junit.framework.TestSuite; +import org.apache.ignite.testframework.GridTestUtils; /** * Cache interceptor suite. @@ -25,41 +27,48 @@ public class IgniteCacheInterceptorSelfTestSuite extends TestSuite { /** * @return Cache API test suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("CacheInterceptor Test Suite"); - suite.addTestSuite(GridCacheInterceptorLocalSelfTest.class); - suite.addTestSuite(GridCacheInterceptorLocalWithStoreSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorLocalSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorLocalWithStoreSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheInterceptorLocalAtomicSelfTest.class); - suite.addTestSuite(GridCacheInterceptorLocalAtomicWithStoreSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorLocalAtomicSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorLocalAtomicWithStoreSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheInterceptorAtomicSelfTest.class); - suite.addTestSuite(GridCacheInterceptorAtomicNearEnabledSelfTest.class); - suite.addTestSuite(GridCacheInterceptorAtomicWithStoreSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorAtomicSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorAtomicNearEnabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorAtomicWithStoreSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheInterceptorAtomicReplicatedSelfTest.class); - suite.addTestSuite(GridCacheInterceptorAtomicWithStoreReplicatedSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorAtomicReplicatedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorAtomicWithStoreReplicatedSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheInterceptorSelfTest.class); - suite.addTestSuite(GridCacheInterceptorNearEnabledSelfTest.class); - suite.addTestSuite(GridCacheInterceptorWithStoreSelfTest.class); - suite.addTestSuite(GridCacheInterceptorReplicatedSelfTest.class); - suite.addTestSuite(GridCacheInterceptorReplicatedWithStoreSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorNearEnabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorWithStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorReplicatedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorReplicatedWithStoreSelfTest.class, ignoredTests); // TODO GG-11141. -// suite.addTestSuite(GridCacheOnCopyFlagTxPartitionedSelfTest.class); -// suite.addTestSuite(GridCacheOnCopyFlagReplicatedSelfTest.class); -// suite.addTestSuite(GridCacheOnCopyFlagLocalSelfTest.class); -// suite.addTestSuite(GridCacheOnCopyFlagAtomicSelfTest.class); +// GridTestUtils.addTestIfNeeded(suite,GridCacheOnCopyFlagTxPartitionedSelfTest.class, ignoredTests); +// GridTestUtils.addTestIfNeeded(suite,GridCacheOnCopyFlagReplicatedSelfTest.class, ignoredTests); +// GridTestUtils.addTestIfNeeded(suite,GridCacheOnCopyFlagLocalSelfTest.class, ignoredTests); +// GridTestUtils.addTestIfNeeded(suite,GridCacheOnCopyFlagAtomicSelfTest.class, ignoredTests); - suite.addTestSuite(CacheInterceptorPartitionCounterRandomOperationsTest.class); - suite.addTestSuite(CacheInterceptorPartitionCounterLocalSanityTest.class); + GridTestUtils.addTestIfNeeded(suite,CacheInterceptorPartitionCounterRandomOperationsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,CacheInterceptorPartitionCounterLocalSanityTest.class, ignoredTests); - suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class); - suite.addTestSuite(GridCacheInterceptorTransactionalRebalanceTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorAtomicRebalanceTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheInterceptorTransactionalRebalanceTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAsyncOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAsyncOperationsTest.java index 0b6d7b1cba115..0416d17c44250 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAsyncOperationsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAsyncOperationsTest.java @@ -37,6 +37,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -76,6 +77,13 @@ public void testAsyncOperationsTx() throws Exception { asyncOperations(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testAsyncOperationsMvccTx() throws Exception { + asyncOperations(TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java index 6210a302f0b67..a5126302685fe 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheAbstractByteArrayValuesSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.jetbrains.annotations.Nullable; @@ -172,6 +173,9 @@ private void testTransaction0(IgniteCache[] caches, Transaction */ private void testTransactionMixed0(IgniteCache[] caches, TransactionConcurrency concurrency, Integer key1, byte[] val1, @Nullable Integer key2, @Nullable Object val2) throws Exception { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, REPEATABLE_READ)) + return; + for (IgniteCache cache : caches) { info("Checking cache: " + cache.getName()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java index 4af2571ada55b..0ee626213b3c4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -31,6 +32,15 @@ */ public abstract class GridCacheAbstractPartitionedByteArrayValuesSelfTest extends GridCacheAbstractDistributedByteArrayValuesSelfTest { + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (nearConfiguration() != null) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheClientModesAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheClientModesAbstractSelfTest.java index 33766f3725986..9316c2dc09149 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheClientModesAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheClientModesAbstractSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -51,6 +52,9 @@ public abstract class GridCacheClientModesAbstractSelfTest extends GridCacheAbst /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + if (nearEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + gridCnt = new AtomicInteger(); super.beforeTestsStarted(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheEventAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheEventAbstractTest.java index daa1557d7273a..9183476aff820 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheEventAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheEventAbstractTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.events.EventType.EVTS_CACHE; @@ -77,6 +78,8 @@ protected boolean partitioned() { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + super.beforeTestsStarted(); gridCnt = gridCount(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMixedModeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMixedModeSelfTest.java index 3d6c08806db2b..bd309fa7d4ea5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMixedModeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMixedModeSelfTest.java @@ -27,6 +27,8 @@ /** * Tests cache puts in mixed mode. + * + * TODO IGNITE-10345: Remove test in ignite 3.0. */ public class GridCacheMixedModeSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java index 912aecea96c8a..0d1aff9216648 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheMultiNodeAbstractTest.java @@ -184,6 +184,8 @@ public void testMultiValueMultiNodePut() throws Exception { * @throws Exception If check fails. */ private void checkPuts(int cnt, Ignite... ignites) throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + CountDownLatch latch = new CountDownLatch(ignites.length * cnt); CacheEventListener lsnr = new CacheEventListener(latch, EVT_CACHE_OBJECT_PUT); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadEventsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadEventsAbstractSelfTest.java index 59af680c80002..01488048898a8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadEventsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePreloadEventsAbstractSelfTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -46,6 +47,13 @@ public abstract class GridCachePreloadEventsAbstractSelfTest extends GridCommonA /** */ private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java index c94457f942a88..b4af2a8ed1d14 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java @@ -186,7 +186,7 @@ private void testLocalTxPreloading(TransactionConcurrency txConcurrency) throws IgniteTransactions txs = ignite(i).transactions(); - try (Transaction tx = txs.txStart(txConcurrency, TransactionIsolation.READ_COMMITTED)) { + try (Transaction tx = txs.txStart(txConcurrency, TransactionIsolation.REPEATABLE_READ)) { cache.invoke(TX_KEY, new EntryProcessor() { @Override public Void process(MutableEntry e, Object... args) { Integer val = e.getValue(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java index 571362143ea5f..bcc72e5b95e70 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionTimeoutException; @@ -55,6 +56,17 @@ public class IgniteTxRemoveTimeoutObjectsTest extends GridCacheAbstractSelfTest return 60_000; } + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7388"); + + if (nearEnabled()) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest.java index 86738f5211662..18459c91011ed 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest.java @@ -34,9 +34,6 @@ public abstract class GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest ex /** Offheap cache name. */ protected static final String CACHE_ATOMIC = "cache_atomic"; - /** Offheap cache name. */ - protected static final String CACHE_ATOMIC_OFFHEAP = "cache_atomic_offheap"; - /** Atomic caches. */ private static IgniteCache[] cachesAtomic; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractTransformWriteThroughSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractTransformWriteThroughSelfTest.java index 34c96fd999394..1c1f3a55f15e7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractTransformWriteThroughSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheAbstractTransformWriteThroughSelfTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -132,6 +133,8 @@ public abstract class GridCacheAbstractTransformWriteThroughSelfTest extends Gri /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + super.beforeTestsStarted(); for (int i = 0; i < GRID_CNT; i++) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOnlyTopologySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOnlyTopologySelfTest.java index 2124bc8ef7e11..4344bb1359213 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOnlyTopologySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearOnlyTopologySelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -53,6 +54,13 @@ public class GridCacheNearOnlyTopologySelfTest extends GridCommonAbstractTest { /** Use cache flag. */ private boolean cache = true; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java index 54eb32d3f1ce6..1c79db7e7a111 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java @@ -21,7 +21,6 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.distributed.GridCacheEventAbstractTest; -import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -31,13 +30,6 @@ * Tests events. */ public class GridCachePartitionedEventSelfTest extends GridCacheEventAbstractTest { - /** {@inheritDoc} */ - @Override public void setUp() throws Exception { - MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); - - super.setUp(); - } - /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { CacheConfiguration cfg = defaultCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePutArrayValueSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePutArrayValueSelfTest.java index 14a0c0f931d38..7ee83e816279c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePutArrayValueSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePutArrayValueSelfTest.java @@ -22,10 +22,12 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheInternal; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -38,6 +40,13 @@ public class GridCachePutArrayValueSelfTest extends GridCacheAbstractSelfTest { return 4; } + @Override protected void initStoreStrategy() throws IgniteCheckedException { + if (!MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + return; + + super.initStoreStrategy(); + } + /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { CacheConfiguration cacheCfg = super.cacheConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxMultiThreadedSelfTest.java new file mode 100644 index 0000000000000..2df5d45b5bfd5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxMultiThreadedSelfTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.replicated; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.IgniteMvccTxMultiThreadedAbstractTest; + +import static org.apache.ignite.cache.CacheMode.REPLICATED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * Tests for replicated transactions. + */ +public class GridCacheReplicatedMvccTxMultiThreadedSelfTest extends IgniteMvccTxMultiThreadedAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setCacheMode(REPLICATED); + ccfg.setEvictionPolicy(null); + + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int keyCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int maxKeyValue() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int threadCount() { + return 5; + } + + /** {@inheritDoc} */ + @Override protected int iterations() { + return 1000; + } + + /** {@inheritDoc} */ + @Override protected boolean isTestDebug() { + return false; + } + + /** {@inheritDoc} */ + @Override protected boolean printMemoryStats() { + return true; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxSingleThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxSingleThreadedSelfTest.java new file mode 100644 index 0000000000000..7e8063431aa45 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxSingleThreadedSelfTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.replicated; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.IgniteMvccTxSingleThreadedAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.REPLICATED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * Tests for replicated transactions. + */ +public class GridCacheReplicatedMvccTxSingleThreadedSelfTest extends IgniteMvccTxSingleThreadedAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + ccfg.setCacheMode(REPLICATED); + ccfg.setEvictionPolicy(null); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int keyCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int maxKeyValue() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected int iterations() { + return 20; + } + + /** {@inheritDoc} */ + @Override protected boolean isTestDebug() { + return false; + } + + /** {@inheritDoc} */ + @Override protected boolean printMemoryStats() { + return true; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxTimeoutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxTimeoutSelfTest.java new file mode 100644 index 0000000000000..2f953236c467b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMvccTxTimeoutSelfTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.replicated; + +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.distributed.IgniteMvccTxTimeoutAbstractTest; + +import static org.apache.ignite.cache.CacheMode.REPLICATED; + +/** + * Simple cache test. + */ +public class GridCacheReplicatedMvccTxTimeoutSelfTest extends IgniteMvccTxTimeoutAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + ccfg.setCacheMode(REPLICATED); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxMultiThreadedSelfTest.java index bcd7e582aeeee..30c79d7f49340 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxMultiThreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxMultiThreadedSelfTest.java @@ -20,12 +20,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; -import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.IgniteTxMultiThreadedAbstractTest; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.log4j.Level; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -34,15 +29,6 @@ * Tests for replicated transactions. */ public class GridCacheReplicatedTxMultiThreadedSelfTest extends IgniteTxMultiThreadedAbstractTest { - /** Cache debug flag. */ - private static final boolean CACHE_DEBUG = false; - - /** Log to file flag. */ - private static final boolean LOG_TO_FILE = true; - - /** */ - private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - /** {@inheritDoc} */ @SuppressWarnings({"unchecked"}) @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -50,8 +36,6 @@ public class GridCacheReplicatedTxMultiThreadedSelfTest extends IgniteTxMultiThr TransactionConfiguration tCfg = new TransactionConfiguration(); - tCfg.setTxSerializableEnabled(true); - c.setTransactionConfiguration(tCfg); CacheConfiguration cc = defaultCacheConfiguration(); @@ -64,15 +48,6 @@ public class GridCacheReplicatedTxMultiThreadedSelfTest extends IgniteTxMultiThr c.setCacheConfiguration(cc); - TcpDiscoverySpi spi = new TcpDiscoverySpi(); - - spi.setIpFinder(ipFinder); - - c.setDiscoverySpi(spi); - - if (CACHE_DEBUG) - resetLog4j(Level.DEBUG, LOG_TO_FILE, GridCacheProcessor.class.getPackage().getName()); - return c; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxSingleThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxSingleThreadedSelfTest.java index 68d8a93e4b44d..80b2f3b4b22fc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxSingleThreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedTxSingleThreadedSelfTest.java @@ -19,12 +19,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.IgniteTxSingleThreadedAbstractTest; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.log4j.Level; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -33,42 +28,19 @@ * Tests for replicated transactions. */ public class GridCacheReplicatedTxSingleThreadedSelfTest extends IgniteTxSingleThreadedAbstractTest { - /** Cache debug flag. */ - private static final boolean CACHE_DEBUG = false; - - /** Log to file flag. */ - private static final boolean LOG_TO_FILE = true; - - /** */ - private TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - - /** {@inheritDoc} */ - @SuppressWarnings({"unchecked"}) + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration c = super.getConfiguration(igniteInstanceName); - - c.getTransactionConfiguration().setTxSerializableEnabled(true); - - CacheConfiguration cc = defaultCacheConfiguration(); - - cc.setCacheMode(REPLICATED); - - cc.setEvictionPolicy(null); - - cc.setWriteSynchronizationMode(FULL_SYNC); - - c.setCacheConfiguration(cc); - - TcpDiscoverySpi spi = new TcpDiscoverySpi(); + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - spi.setIpFinder(ipFinder); + CacheConfiguration ccfg = defaultCacheConfiguration(); - c.setDiscoverySpi(spi); + ccfg.setCacheMode(REPLICATED); + ccfg.setEvictionPolicy(null); + ccfg.setWriteSynchronizationMode(FULL_SYNC); - if (CACHE_DEBUG) - resetLog4j(Level.DEBUG, LOG_TO_FILE, GridCacheProcessor.class.getPackage().getName()); + cfg.setCacheConfiguration(ccfg); - return c; + return cfg; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadSelfTest.java index f1140916d507d..8101172e45aeb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadSelfTest.java @@ -17,12 +17,9 @@ package org.apache.ignite.internal.processors.cache.distributed.replicated.preloader; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Random; import java.util.UUID; @@ -39,10 +36,7 @@ import org.apache.ignite.cache.CacheEntryEventSerializableFilter; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.CacheRebalanceMode; -import org.apache.ignite.cache.affinity.AffinityFunction; -import org.apache.ignite.cache.affinity.AffinityFunctionContext; import org.apache.ignite.cache.affinity.AffinityKeyMapper; -import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.Event; @@ -51,15 +45,17 @@ import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.P2; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.plugin.CachePluginConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -84,9 +80,6 @@ public class GridCacheReplicatedPreloadSelfTest extends GridCommonAbstractTest { /** */ private int batchSize = 4096; - /** */ - private int poolSize = 2; - /** */ private volatile boolean extClassloadingAtCfg = false; @@ -119,6 +112,8 @@ public class GridCacheReplicatedPreloadSelfTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + cfg.setRebalanceThreadPoolSize(2); + TcpDiscoverySpi disco = new TcpDiscoverySpi(); disco.setIpFinder(ipFinder); @@ -173,7 +168,6 @@ CacheConfiguration cacheConfiguration(String igniteInstanceName) { cacheCfg.setWriteSynchronizationMode(FULL_SYNC); cacheCfg.setRebalanceMode(preloadMode); cacheCfg.setRebalanceBatchSize(batchSize); - cacheCfg.setRebalanceThreadPoolSize(poolSize); if (extClassloadingAtCfg) loadExternalClassesToCfg(cacheCfg); @@ -571,14 +565,17 @@ public void testExternalClassesAtMessage() throws Exception { * @throws Exception If test failed. */ public void testExternalClassesAtEventP2pDisabled() throws Exception { - testExternalClassesAtEvent0(true); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + testExternalClassesAtEvent0(true); } /** * @throws Exception If test failed. */ public void testExternalClassesAtEvent() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + testExternalClassesAtEvent0(false); } @@ -656,51 +653,30 @@ public void testAsync() throws Exception { try { IgniteCache cache1 = startGrid(1).cache(DEFAULT_CACHE_NAME); - int keyCnt = 2000; + final int keyCnt = 2000; for (int i = 0; i < keyCnt; i++) cache1.put(i, "val" + i); - IgniteCache cache2 = startGrid(2).cache(DEFAULT_CACHE_NAME); + final IgniteCache cache2 = startGrid(2).cache(DEFAULT_CACHE_NAME); int size = cache2.localSize(CachePeekMode.ALL); info("Size of cache2: " + size); - assert waitCacheSize(cache2, keyCnt, getTestTimeout()) : - "Actual cache size: " + cache2.localSize(CachePeekMode.ALL); + boolean awaitSize = GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return cache2.localSize(CachePeekMode.ALL) >= keyCnt; + } + }, getTestTimeout()); + + assertTrue("Actual cache size: " + cache2.localSize(CachePeekMode.ALL), awaitSize); } finally { stopAllGrids(); } } - /** - * @param cache Cache. - * @param expSize Lower bound of expected size. - * @param timeout Timeout. - * @return {@code true} if success. - * @throws InterruptedException If thread was interrupted. - */ - @SuppressWarnings({"BusyWait"}) - private boolean waitCacheSize(IgniteCache cache, int expSize, long timeout) - throws InterruptedException { - assert cache != null; - assert expSize > 0; - assert timeout >= 0; - - long end = System.currentTimeMillis() + timeout; - - while (cache.localSize(CachePeekMode.ALL) < expSize) { - Thread.sleep(50); - - if (end - System.currentTimeMillis() <= 0) - break; - } - - return cache.localSize(CachePeekMode.ALL) >= expSize; - } - /** * @throws Exception If test failed. */ @@ -788,7 +764,7 @@ public void testMultipleNodes() throws Exception { info("Beginning data population..."); - int cnt = 2500; + final int cnt = 2500; Map map = null; @@ -816,7 +792,7 @@ assert grid(gridIdx).cache(DEFAULT_CACHE_NAME).localSize(CachePeekMode.ALL) == c info("Cache size is OK for grid index: " + gridIdx); } - IgniteCache lastCache = startGrid(gridCnt).cache(DEFAULT_CACHE_NAME); + final IgniteCache lastCache = startGrid(gridCnt).cache(DEFAULT_CACHE_NAME); // Let preloading start. Thread.sleep(1000); @@ -828,8 +804,15 @@ assert grid(gridIdx).cache(DEFAULT_CACHE_NAME).localSize(CachePeekMode.ALL) == c stopGrid(idx); - assert waitCacheSize(lastCache, cnt, 20 * 1000) : - "Actual cache size: " + lastCache.localSize(CachePeekMode.ALL); + awaitPartitionMapExchange(true, true, null); + + boolean awaitSize = GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return lastCache.localSize(CachePeekMode.ALL) >= cnt; + } + }, 20_000); + + assertTrue("Actual cache size: " + lastCache.localSize(CachePeekMode.ALL), awaitSize); } finally { stopAllGrids(); @@ -866,65 +849,6 @@ public void testConcurrentStartAsync() throws Exception { } } - /** - * Test affinity. - */ - @SuppressWarnings({"PublicInnerClass"}) - private static class TestAffinityFunction implements AffinityFunction { - /** {@inheritDoc} */ - @Override public int partitions() { - return 2; - } - - /** {@inheritDoc} */ - @Override public int partition(Object key) { - if (key instanceof Number) - return ((Number)key).intValue() % 2; - - return key == null ? 0 : U.safeAbs(key.hashCode() % 2); - } - - /** {@inheritDoc} */ - @Override public List> assignPartitions(AffinityFunctionContext affCtx) { - List> res = new ArrayList<>(partitions()); - - for (int part = 0; part < partitions(); part++) - res.add(nodes(part, affCtx.currentTopologySnapshot())); - - return res; - } - - /** {@inheritDoc} */ - @SuppressWarnings({"RedundantTypeArguments"}) - public List nodes(int part, Collection nodes) { - Collection col = new HashSet<>(nodes); - - if (col.size() <= 1) - return new ArrayList<>(col); - - for (Iterator iter = col.iterator(); iter.hasNext(); ) { - ClusterNode node = iter.next(); - - boolean even = node.attribute("EVEN"); - - if ((even && part != 0) || (!even && part != 1)) - iter.remove(); - } - - return new ArrayList<>(col); - } - - /** {@inheritDoc} */ - @Override public void reset() { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void removeNode(UUID nodeId) { - // No-op. - } - } - /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheDaemonNodeLocalSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheDaemonNodeLocalSelfTest.java index 8578db7306232..aa63fe67abb7b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheDaemonNodeLocalSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheDaemonNodeLocalSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.GridCacheDaemonNodeAbstractSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -26,6 +27,13 @@ * Tests local cache with daemon node. */ public class GridCacheDaemonNodeLocalSelfTest extends GridCacheDaemonNodeAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java index a6be82b58d651..e6c4771a90aad 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java @@ -26,6 +26,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheAbstractByteArrayValuesSelfTest; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.jetbrains.annotations.Nullable; @@ -67,6 +68,8 @@ public class GridCacheLocalByteArrayValuesSelfTest extends GridCacheAbstractByte /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + ignite = startGrid(1); cache = ignite.cache(CACHE_REGULAR); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java index c0b60718001db..716e5dd510f2e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalEventSelfTest.java @@ -30,7 +30,7 @@ public class GridCacheLocalEventSelfTest extends GridCacheEventAbstractTest { /** {@inheritDoc} */ @Override public void beforeTestsStarted() throws Exception { MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); - + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); super.beforeTestsStarted(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java index 56ee7604b521d..40677cbccb000 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreAbstractTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.Nullable; @@ -56,6 +57,13 @@ public abstract class GridCacheWriteBehindStoreAbstractTest extends GridCommonAb /** Cache store. */ private static final GridCacheTestStore store = new GridCacheTestStore(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreLocalTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreLocalTest.java index 59dd4b41930bf..160203e538da0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreLocalTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreLocalTest.java @@ -18,11 +18,19 @@ package org.apache.ignite.internal.processors.cache.store; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Tests {@link org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore} in grid configuration. */ public class GridCacheWriteBehindStoreLocalTest extends GridCacheWriteBehindStoreAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return CacheMode.LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStorePartitionedMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStorePartitionedMultiNodeSelfTest.java index de61058f0a4b8..d60dc4a5c2097 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStorePartitionedMultiNodeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStorePartitionedMultiNodeSelfTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -92,6 +93,13 @@ public class GridCacheWriteBehindStorePartitionedMultiNodeSelfTest extends GridC return c; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { stores = null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreAbstractTest.java index a64104ea56ed6..23bb781327bcd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreAbstractTest.java @@ -28,11 +28,19 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Tests that write behind store is updated if client does not have store. */ public abstract class IgnteCacheClientWriteBehindStoreAbstractTest extends IgniteCacheAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 3; diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index ee9b02af08e00..2ea77b44aee45 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -41,7 +41,8 @@ public enum Feature { ENTRY_LOCK, CACHE_EVENTS, EVICTION, - EXPIRATION + EXPIRATION, + INTERCEPTOR } /** @@ -132,6 +133,9 @@ private static void validateFeature(Feature feature) { case EXPIRATION: fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + + case INTERCEPTOR: + fail("https://issues.apache.org/jira/browse/IGNITE-9323"); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsCacheTestSuite3.java index 0007813ac92c3..8ab7c2c4f2bbf 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsCacheTestSuite3.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsCacheTestSuite3.java @@ -17,9 +17,11 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.binary.GridCacheBinaryAtomicEntryProcessorDeploymentSelfTest; import org.apache.ignite.internal.processors.cache.binary.GridCacheBinaryTransactionalEntryProcessorDeploymentSelfTest; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.config.GridTestProperties; /** @@ -43,14 +45,22 @@ public class IgniteBinaryObjectsCacheTestSuite3 { * @return Test suite. * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { GridTestProperties.setProperty(GridTestProperties.ENTRY_PROCESSOR_CLASS_NAME, "org.apache.ignite.tests.p2p.CacheDeploymentBinaryEntryProcessor"); - TestSuite suite = IgniteCacheTestSuite3.suite(); + TestSuite suite = IgniteCacheTestSuite3.suite(ignoredTests); - suite.addTestSuite(GridCacheBinaryAtomicEntryProcessorDeploymentSelfTest.class); - suite.addTestSuite(GridCacheBinaryTransactionalEntryProcessorDeploymentSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheBinaryAtomicEntryProcessorDeploymentSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheBinaryTransactionalEntryProcessorDeploymentSelfTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite3.java new file mode 100644 index 0000000000000..55dddbed34855 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite3.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.processors.cache.CacheInterceptorPartitionCounterLocalSanityTest; +import org.apache.ignite.internal.processors.cache.CacheInterceptorPartitionCounterRandomOperationsTest; +import org.apache.ignite.internal.processors.cache.GridCacheAtomicEntryProcessorDeploymentSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheEntryVersionSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicNearEnabledSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicRebalanceTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicReplicatedSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicWithStoreReplicatedSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicWithStoreSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorLocalAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheInterceptorLocalAtomicWithStoreSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheValueBytesPreloadingSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheVersionSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheVersionTopologyChangeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheGroupsTest; +import org.apache.ignite.internal.processors.cache.binary.GridCacheBinaryAtomicEntryProcessorDeploymentSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheAsyncOperationsTest; +import org.apache.ignite.internal.processors.cache.distributed.GridCacheMixedModeSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheClientOnlySelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteTxReentryColocatedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheValueConsistencyAtomicNearEnabledSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCacheValueConsistencyAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlySelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMvccTxMultiThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMvccTxSingleThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMvccTxTimeoutSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxMultiThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxSingleThreadedSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxTimeoutSelfTest; +import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStoreMultithreadedSelfTest; +import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStoreSelfTest; +import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreAtomicTest; +import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreNonCoalescingTest; + +/** + * Test suite. + */ +public class IgniteCacheMvccTestSuite3 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + HashSet ignoredTests = new HashSet<>(); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(GridCacheEntryVersionSelfTest.class); + ignoredTests.add(GridCacheVersionTopologyChangeTest.class); + ignoredTests.add(CacheAsyncOperationsTest.class); + ignoredTests.add(CacheInterceptorPartitionCounterLocalSanityTest.class); + ignoredTests.add(CacheInterceptorPartitionCounterRandomOperationsTest.class); + ignoredTests.add(IgniteCacheGroupsTest.class); + + // Atomic caches + ignoredTests.add(GridCacheValueConsistencyAtomicSelfTest.class); + ignoredTests.add(GridCacheValueConsistencyAtomicNearEnabledSelfTest.class); + ignoredTests.add(GridCacheReplicatedAtomicGetAndTransformStoreSelfTest.class); + ignoredTests.add(GridCacheAtomicEntryProcessorDeploymentSelfTest.class); + ignoredTests.add(GridCacheValueBytesPreloadingSelfTest.class); + ignoredTests.add(GridCacheBinaryAtomicEntryProcessorDeploymentSelfTest.class); + + ignoredTests.add(GridCacheClientOnlySelfTest.CasePartitionedAtomic.class); + ignoredTests.add(GridCacheClientOnlySelfTest.CaseReplicatedAtomic.class); + ignoredTests.add(GridCacheNearOnlySelfTest.CasePartitionedAtomic.class); + ignoredTests.add(GridCacheNearOnlySelfTest.CaseReplicatedAtomic.class); + + ignoredTests.add(IgnteCacheClientWriteBehindStoreAtomicTest.class); + ignoredTests.add(IgnteCacheClientWriteBehindStoreNonCoalescingTest.class); + + ignoredTests.add(GridCacheInterceptorLocalAtomicSelfTest.class); + ignoredTests.add(GridCacheInterceptorLocalAtomicWithStoreSelfTest.class); + ignoredTests.add(GridCacheInterceptorAtomicSelfTest.class); + ignoredTests.add(GridCacheInterceptorAtomicNearEnabledSelfTest.class); + ignoredTests.add(GridCacheInterceptorAtomicWithStoreSelfTest.class); + ignoredTests.add(GridCacheInterceptorAtomicReplicatedSelfTest.class); + ignoredTests.add(GridCacheInterceptorAtomicWithStoreReplicatedSelfTest.class); + ignoredTests.add(GridCacheInterceptorAtomicRebalanceTest.class); + + // Irrelevant tx tests + ignoredTests.add(IgniteTxReentryNearSelfTest.class); + ignoredTests.add(IgniteTxReentryColocatedSelfTest.class); + + // Other non-tx tests + ignoredTests.add(GridCacheWriteBehindStoreSelfTest.class); + ignoredTests.add(GridCacheWriteBehindStoreMultithreadedSelfTest.class); + + ignoredTests.add(GridCacheVersionSelfTest.class); + ignoredTests.add(GridCacheMixedModeSelfTest.class); + + // Skip classes which Mvcc implementations are added in this method below. + // TODO IGNITE-10175: refactor these tests (use assume) to support both mvcc and non-mvcc modes after moving to JUnit4/5. + ignoredTests.add(GridCacheReplicatedTxSingleThreadedSelfTest.class); // See GridCacheReplicatedMvccTxSingleThreadedSelfTest + ignoredTests.add(GridCacheReplicatedTxMultiThreadedSelfTest.class); // See GridCacheReplicatedMvccTxMultiThreadedSelfTest + ignoredTests.add(GridCacheReplicatedTxTimeoutSelfTest.class); // See GridCacheReplicatedMvccTxTimeoutSelfTest + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 3"); + + suite.addTest(IgniteBinaryObjectsCacheTestSuite3.suite(ignoredTests)); + + // Add Mvcc clones. + suite.addTestSuite(GridCacheReplicatedMvccTxSingleThreadedSelfTest.class); + suite.addTestSuite(GridCacheReplicatedMvccTxMultiThreadedSelfTest.class); + suite.addTestSuite(GridCacheReplicatedMvccTxTimeoutSelfTest.class); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheNearOnlySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheNearOnlySelfTestSuite.java index 087007e9fb120..1519baa9ac4ac 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheNearOnlySelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheNearOnlySelfTestSuite.java @@ -17,10 +17,12 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheClientOnlySelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlySelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyTopologySelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite for near-only cache. @@ -28,22 +30,29 @@ public class IgniteCacheNearOnlySelfTestSuite { /** * @return Suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Near-only cache test suite."); - suite.addTest(new TestSuite(GridCacheClientOnlySelfTest.CasePartitionedAtomic.class)); - suite.addTest(new TestSuite(GridCacheClientOnlySelfTest.CasePartitionedTransactional.class)); - suite.addTest(new TestSuite(GridCacheClientOnlySelfTest.CaseReplicatedAtomic.class)); - suite.addTest(new TestSuite(GridCacheClientOnlySelfTest.CaseReplicatedTransactional.class)); + GridTestUtils.addTestIfNeeded(suite,GridCacheClientOnlySelfTest.CasePartitionedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheClientOnlySelfTest.CasePartitionedTransactional.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheClientOnlySelfTest.CaseReplicatedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheClientOnlySelfTest.CaseReplicatedTransactional.class, ignoredTests); - suite.addTest(new TestSuite(GridCacheNearOnlyTopologySelfTest.class)); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearOnlyTopologySelfTest.class, ignoredTests); - suite.addTest(new TestSuite(GridCacheNearOnlySelfTest.CasePartitionedAtomic.class)); - suite.addTest(new TestSuite(GridCacheNearOnlySelfTest.CasePartitionedTransactional.class)); - suite.addTest(new TestSuite(GridCacheNearOnlySelfTest.CaseReplicatedAtomic.class)); - suite.addTest(new TestSuite(GridCacheNearOnlySelfTest.CaseReplicatedTransactional.class)); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearOnlySelfTest.CasePartitionedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearOnlySelfTest.CasePartitionedTransactional.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearOnlySelfTest.CaseReplicatedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearOnlySelfTest.CaseReplicatedTransactional.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index 179f5e95cff35..0250d58d386a6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -17,7 +17,7 @@ package org.apache.ignite.testsuites; -import java.util.HashSet; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.cache.affinity.rendezvous.ClusterNodeAttributeAffinityBackupFilterSelfTest; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunctionBackupFilterSelfTest; @@ -153,7 +153,6 @@ public class IgniteCacheTestSuite2 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ public static TestSuite suite() { return suite(null); @@ -162,9 +161,8 @@ public static TestSuite suite() { /** * @param ignoredTests Ignored tests. * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite(HashSet ignoredTests) { + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 2"); // Local cache. diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java index 6c59c4c19a485..63b93e095b501 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.CacheStartupInDeploymentModesTest; import org.apache.ignite.internal.processors.cache.GridCacheAtomicEntryProcessorDeploymentSelfTest; @@ -77,12 +78,12 @@ import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxSingleThreadedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxTimeoutSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheSyncReplicatedPreloadSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.replicated.GridReplicatedTxPreloadTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadLifecycleSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadStartStopEventsSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheDaemonNodeLocalSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalByteArrayValuesSelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -90,110 +91,116 @@ public class IgniteCacheTestSuite3 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 3"); - suite.addTestSuite(IgniteCacheGroupsTest.class); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheGroupsTest.class, ignoredTests); // Value consistency tests. - suite.addTestSuite(GridCacheValueConsistencyAtomicSelfTest.class); - suite.addTestSuite(GridCacheValueConsistencyAtomicNearEnabledSelfTest.class); - suite.addTestSuite(GridCacheValueConsistencyTransactionalSelfTest.class); - suite.addTestSuite(GridCacheValueConsistencyTransactionalNearEnabledSelfTest.class); - suite.addTestSuite(GridCacheValueBytesPreloadingSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheValueConsistencyAtomicSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheValueConsistencyAtomicNearEnabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheValueConsistencyTransactionalSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheValueConsistencyTransactionalNearEnabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheValueBytesPreloadingSelfTest.class, ignoredTests); // Replicated cache. - suite.addTestSuite(GridCacheReplicatedBasicApiTest.class); - suite.addTestSuite(GridCacheReplicatedBasicOpSelfTest.class); - suite.addTestSuite(GridCacheReplicatedBasicStoreSelfTest.class); - suite.addTestSuite(GridCacheReplicatedGetAndTransformStoreSelfTest.class); - suite.addTestSuite(GridCacheReplicatedAtomicGetAndTransformStoreSelfTest.class); - suite.addTestSuite(GridCacheReplicatedEventSelfTest.class); - suite.addTestSuite(GridCacheReplicatedEventDisabledSelfTest.class); - suite.addTestSuite(GridCacheReplicatedSynchronousCommitTest.class); - - suite.addTestSuite(GridCacheReplicatedLockSelfTest.class); - suite.addTestSuite(GridCacheReplicatedMultiNodeLockSelfTest.class); - suite.addTestSuite(GridCacheReplicatedMultiNodeSelfTest.class); - suite.addTestSuite(GridCacheReplicatedNodeFailureSelfTest.class); - suite.addTestSuite(GridCacheReplicatedTxSingleThreadedSelfTest.class); - suite.addTestSuite(GridCacheReplicatedTxTimeoutSelfTest.class); - suite.addTestSuite(GridCacheReplicatedPreloadSelfTest.class); - suite.addTestSuite(GridCacheReplicatedPreloadLifecycleSelfTest.class); - suite.addTestSuite(GridCacheSyncReplicatedPreloadSelfTest.class); - - //suite.addTestSuite(GridCacheReplicatedEntrySetSelfTest.class); - //suite.addTestSuite(GridCacheReplicatedMarshallerTxTest.class); - //suite.addTestSuite(GridCacheReplicatedOnheapFullApiSelfTest.class); - //suite.addTestSuite(GridCacheReplicatedOnheapMultiNodeFullApiSelfTest.class); - //suite.addTestSuite(GridCacheReplicatedTxConcurrentGetTest.class); - //suite.addTestSuite(GridCacheReplicatedTxMultiNodeBasicTest.class); - //suite.addTestSuite(GridCacheReplicatedTxReadTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedBasicApiTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedBasicOpSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedBasicStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedGetAndTransformStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedAtomicGetAndTransformStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedEventDisabledSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedSynchronousCommitTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedLockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedMultiNodeLockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedNodeFailureSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedTxSingleThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedTxTimeoutSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedPreloadSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedPreloadLifecycleSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheSyncReplicatedPreloadSelfTest.class, ignoredTests); + + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedEntrySetSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedMarshallerTxTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedOnheapFullApiSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedOnheapMultiNodeFullApiSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedTxConcurrentGetTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedTxMultiNodeBasicTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedTxReadTest.class, ignoredTests); // TODO GG-11141. -// suite.addTestSuite(GridCacheDeploymentSelfTest.class); -// suite.addTestSuite(GridCacheDeploymentOffHeapSelfTest.class); -// suite.addTestSuite(GridCacheDeploymentOffHeapValuesSelfTest.class); - suite.addTestSuite(CacheStartupInDeploymentModesTest.class); - suite.addTestSuite(GridCacheConditionalDeploymentSelfTest.class); - suite.addTestSuite(GridCacheAtomicEntryProcessorDeploymentSelfTest.class); - suite.addTestSuite(GridCacheTransactionalEntryProcessorDeploymentSelfTest.class); - suite.addTestSuite(IgniteCacheScanPredicateDeploymentSelfTest.class); - - suite.addTestSuite(GridCachePutArrayValueSelfTest.class); - suite.addTestSuite(GridCacheReplicatedEvictionEventSelfTest.class); - suite.addTestSuite(GridCacheReplicatedTxMultiThreadedSelfTest.class); - suite.addTestSuite(GridCacheReplicatedPreloadEventsSelfTest.class); - suite.addTestSuite(GridCacheReplicatedPreloadStartStopEventsSelfTest.class); - suite.addTestSuite(GridReplicatedTxPreloadTest.class); - - suite.addTestSuite(IgniteTxReentryNearSelfTest.class); - suite.addTestSuite(IgniteTxReentryColocatedSelfTest.class); +// GridTestUtils.addTestIfNeeded(suite,GridCacheDeploymentSelfTest.class, ignoredTests); +// GridTestUtils.addTestIfNeeded(suite,GridCacheDeploymentOffHeapSelfTest.class, ignoredTests); +// GridTestUtils.addTestIfNeeded(suite,GridCacheDeploymentOffHeapValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,CacheStartupInDeploymentModesTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheConditionalDeploymentSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheAtomicEntryProcessorDeploymentSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheTransactionalEntryProcessorDeploymentSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheScanPredicateDeploymentSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite,GridCachePutArrayValueSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedEvictionEventSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedTxMultiThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedPreloadEventsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedPreloadStartStopEventsSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite,IgniteTxReentryNearSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteTxReentryColocatedSelfTest.class, ignoredTests); // Test for byte array value special case. - suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCacheNearPartitionedP2PEnabledByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCacheNearPartitionedP2PDisabledByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCachePartitionedOnlyP2PEnabledByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCachePartitionedOnlyP2PDisabledByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCacheReplicatedP2PEnabledByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCacheReplicatedP2PDisabledByteArrayValuesSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheLocalByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearPartitionedP2PEnabledByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheNearPartitionedP2PDisabledByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedOnlyP2PEnabledByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedOnlyP2PDisabledByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedP2PEnabledByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedP2PDisabledByteArrayValuesSelfTest.class, ignoredTests); // Near-only cache. - suite.addTest(IgniteCacheNearOnlySelfTestSuite.suite()); + suite.addTest(IgniteCacheNearOnlySelfTestSuite.suite(ignoredTests)); // Test cache with daemon nodes. - suite.addTestSuite(GridCacheDaemonNodeLocalSelfTest.class); - suite.addTestSuite(GridCacheDaemonNodePartitionedSelfTest.class); - suite.addTestSuite(GridCacheDaemonNodeReplicatedSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheDaemonNodeLocalSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheDaemonNodePartitionedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheDaemonNodeReplicatedSelfTest.class, ignoredTests); // Write-behind. - suite.addTest(IgniteCacheWriteBehindTestSuite.suite()); + suite.addTest(IgniteCacheWriteBehindTestSuite.suite(ignoredTests)); // Transform. - suite.addTestSuite(GridCachePartitionedTransformWriteThroughBatchUpdateSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedTransformWriteThroughBatchUpdateSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheEntryVersionSelfTest.class); - suite.addTestSuite(GridCacheVersionSelfTest.class); - suite.addTestSuite(GridCacheVersionTopologyChangeTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheEntryVersionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheVersionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheVersionTopologyChangeTest.class, ignoredTests); // Memory leak tests. - suite.addTestSuite(GridCacheReferenceCleanupSelfTest.class); - suite.addTestSuite(GridCacheReloadSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheReferenceCleanupSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,GridCacheReloadSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheMixedModeSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheMixedModeSelfTest.class, ignoredTests); // Cache interceptor tests. - suite.addTest(IgniteCacheInterceptorSelfTestSuite.suite()); + suite.addTest(IgniteCacheInterceptorSelfTestSuite.suite(ignoredTests)); - suite.addTestSuite(IgniteTxGetAfterStopTest.class); + GridTestUtils.addTestIfNeeded(suite,IgniteTxGetAfterStopTest.class, ignoredTests); - suite.addTestSuite(CacheAsyncOperationsTest.class); + GridTestUtils.addTestIfNeeded(suite,CacheAsyncOperationsTest.class, ignoredTests); - suite.addTestSuite(IgniteTxRemoveTimeoutObjectsTest.class); - suite.addTestSuite(IgniteTxRemoveTimeoutObjectsNearTest.class); + GridTestUtils.addTestIfNeeded(suite,IgniteTxRemoveTimeoutObjectsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteTxRemoveTimeoutObjectsNearTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java index dff93ffa5ecf0..b4f6ae775ba9b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheWriteBehindTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.GridCachePartitionedWritesTest; import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStoreLocalTest; @@ -28,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreAtomicTest; import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreNonCoalescingTest; import org.apache.ignite.internal.processors.cache.store.IgnteCacheClientWriteBehindStoreTxTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite that contains all tests for {@link org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore}. @@ -35,22 +37,29 @@ public class IgniteCacheWriteBehindTestSuite extends TestSuite { /** * @return Ignite Bamboo in-memory data grid test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Write-Behind Store Test Suite"); // Write-behind tests. - suite.addTest(new TestSuite(GridCacheWriteBehindStoreSelfTest.class)); - suite.addTest(new TestSuite(GridCacheWriteBehindStoreMultithreadedSelfTest.class)); - suite.addTest(new TestSuite(GridCacheWriteBehindStoreLocalTest.class)); - suite.addTest(new TestSuite(GridCacheWriteBehindStoreReplicatedTest.class)); - suite.addTest(new TestSuite(GridCacheWriteBehindStorePartitionedTest.class)); - suite.addTest(new TestSuite(GridCacheWriteBehindStorePartitionedMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedWritesTest.class)); - suite.addTest(new TestSuite(IgnteCacheClientWriteBehindStoreAtomicTest.class)); - suite.addTest(new TestSuite(IgnteCacheClientWriteBehindStoreTxTest.class)); - suite.addTest(new TestSuite(IgnteCacheClientWriteBehindStoreNonCoalescingTest.class)); + GridTestUtils.addTestIfNeeded(suite, GridCacheWriteBehindStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheWriteBehindStoreMultithreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheWriteBehindStoreLocalTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheWriteBehindStoreReplicatedTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheWriteBehindStorePartitionedTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheWriteBehindStorePartitionedMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedWritesTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgnteCacheClientWriteBehindStoreAtomicTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgnteCacheClientWriteBehindStoreTxTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgnteCacheClientWriteBehindStoreNonCoalescingTest.class, ignoredTests); return suite; } From 500f768ba24aefe11cb1ddbc7943d6624005ce95 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Fri, 30 Nov 2018 15:06:48 +0300 Subject: [PATCH 192/403] IGNITE-10050: MVCC: Create "Cache 5" test suite for MVCC mode. This closes #5474. (cherry picked from commit bfe9283e9e809cad21b26466bcb8fcb7d7244b73) --- .../cache/CacheKeepBinaryTransactionTest.java | 8 +- .../cache/CacheNearReaderUpdateTest.java | 6 + .../cache/CacheRebalancingSelfTest.java | 13 +- .../ClusterReadOnlyModeAbstractTest.java | 11 +- ...ntryVersionConsistencyReadThroughTest.java | 22 +++ ...gniteCacheReadThroughEvictionSelfTest.java | 14 ++ ...heReadThroughEvictionsVariationsSuite.java | 3 +- .../cache/IgniteCacheStoreCollectionTest.java | 28 +++- .../CacheLateAffinityAssignmentTest.java | 4 + .../dht/NotMappedPartitionInTxTest.java | 149 +++++++----------- .../CacheManualRebalancingTest.java | 7 +- .../testsuites/IgniteCacheMvccTestSuite2.java | 3 +- .../testsuites/IgniteCacheMvccTestSuite5.java | 94 +++++++++++ .../testsuites/IgniteCacheTestSuite5.java | 79 ++++++---- 14 files changed, 290 insertions(+), 151 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java index 841e49c7c1a66..43e0578158719 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheKeepBinaryTransactionTest.java @@ -26,6 +26,7 @@ import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -40,8 +41,11 @@ public class CacheKeepBinaryTransactionTest extends GridCommonAbstractTest { IgniteConfiguration cfg = super.getConfiguration(gridName); TransactionConfiguration txCfg = new TransactionConfiguration(); - txCfg.setDefaultTxConcurrency(TransactionConcurrency.OPTIMISTIC); - txCfg.setDefaultTxIsolation(TransactionIsolation.REPEATABLE_READ); + + if (!MvccFeatureChecker.forcedMvcc()) { + txCfg.setDefaultTxConcurrency(TransactionConcurrency.OPTIMISTIC); + txCfg.setDefaultTxIsolation(TransactionIsolation.REPEATABLE_READ); + } cfg.setTransactionConfiguration(txCfg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNearReaderUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNearReaderUpdateTest.java index 96167f5d038dc..e47ebb312ecad 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNearReaderUpdateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNearReaderUpdateTest.java @@ -43,6 +43,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -91,6 +92,8 @@ public class CacheNearReaderUpdateTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + super.beforeTestsStarted(); startGridsMultiThreaded(SRVS); @@ -336,6 +339,9 @@ private CacheConfiguration cacheConfiguration( int backups, boolean storeEnabled, boolean nearCache) { + if (storeEnabled) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); ccfg.setCacheMode(cacheMode); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java index 75e62964b3ac8..469a676f4a03d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -48,12 +49,12 @@ public class CacheRebalancingSelfTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - CacheConfiguration rebalabceCacheCfg = new CacheConfiguration<>(); - rebalabceCacheCfg.setBackups(1); - rebalabceCacheCfg.setName(REBALANCE_TEST_CACHE_NAME); - rebalabceCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + CacheConfiguration ccfg = new CacheConfiguration<>(); + ccfg.setBackups(1); + ccfg.setName(REBALANCE_TEST_CACHE_NAME); + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME), rebalabceCacheCfg); + cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME), ccfg); return cfg; } @@ -69,6 +70,8 @@ public class CacheRebalancingSelfTest extends GridCommonAbstractTest { * @throws Exception If fails. */ public void testRebalanceLocalCacheFuture() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + startGrid( getTestIgniteInstanceName(0), getConfiguration(getTestIgniteInstanceName(0)) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java index 7cb9649a4ffb8..3134f1c4ff1f3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java @@ -30,6 +30,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -46,12 +47,18 @@ public class ClusterReadOnlyModeAbstractTest extends GridCommonAbstractTest { /** Replicated transactional cache. */ private static final String REPL_TX_CACHE = "repl_tx_cache"; + /** Replicated transactional cache. */ + private static final String REPL_MVCC_CACHE = "repl_mvcc_cache"; + /** Partitioned atomic cache. */ private static final String PART_ATOMIC_CACHE = "part_atomic_cache"; /** Partitioned transactional cache. */ private static final String PART_TX_CACHE = "part_tx_cache"; + /** Partitioned mvcc transactional cache. */ + private static final String PART_MVCC_CACHE = "part_mvcc_cache"; + /** Cache names. */ protected static final Collection CACHE_NAMES = F.asList(REPL_ATOMIC_CACHE, REPL_TX_CACHE, PART_ATOMIC_CACHE, PART_TX_CACHE); @@ -77,8 +84,10 @@ public class ClusterReadOnlyModeAbstractTest extends GridCommonAbstractTest { cfg.setCacheConfiguration( cacheConfiguration(REPL_ATOMIC_CACHE, REPLICATED, ATOMIC, null), cacheConfiguration(REPL_TX_CACHE, REPLICATED, TRANSACTIONAL, null), + cacheConfiguration(REPL_MVCC_CACHE, REPLICATED, TRANSACTIONAL_SNAPSHOT, "mvcc_repl_grp"), cacheConfiguration(PART_ATOMIC_CACHE, PARTITIONED, ATOMIC, "part_grp"), - cacheConfiguration(PART_TX_CACHE, PARTITIONED, TRANSACTIONAL, "part_grp") + cacheConfiguration(PART_TX_CACHE, PARTITIONED, TRANSACTIONAL, "part_grp"), + cacheConfiguration(PART_MVCC_CACHE, PARTITIONED, TRANSACTIONAL_SNAPSHOT, "mvcc_part_grp") ); return cfg; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/EntryVersionConsistencyReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/EntryVersionConsistencyReadThroughTest.java index ebf253d50cd73..9e6f627e68424 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/EntryVersionConsistencyReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/EntryVersionConsistencyReadThroughTest.java @@ -38,10 +38,12 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -106,6 +108,16 @@ public void testInvokeAllTransactionalCache() throws Exception { check(false, createCacheConfiguration(TRANSACTIONAL)); } + /** + * @throws Exception If failed. + */ + public void testInvokeAllMvccTxCache() throws Exception { + if (!MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + check(false, createCacheConfiguration(TRANSACTIONAL_SNAPSHOT)); + } + /** * @throws Exception If failed. */ @@ -127,6 +139,16 @@ public void testInvokeTransactionalCache() throws Exception { check(true, createCacheConfiguration(TRANSACTIONAL)); } + /** + * @throws Exception If failed. + */ + public void testInvokeMvccTxCache() throws Exception { + if (!MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + check(true, createCacheConfiguration(TRANSACTIONAL_SNAPSHOT)); + } + /** * Tests entry's versions consistency after invokeAll. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionSelfTest.java index 00431b195a021..fe186b7908423 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionSelfTest.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.IgniteCacheConfigVariationsAbstractTest; import javax.cache.configuration.Factory; @@ -48,6 +49,13 @@ public class IgniteCacheReadThroughEvictionSelfTest extends IgniteCacheConfigVar /** */ private static final int KEYS = 100; + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { storeStgy.resetStore(); @@ -57,6 +65,8 @@ public class IgniteCacheReadThroughEvictionSelfTest extends IgniteCacheConfigVar * @throws Exception if failed. */ public void testReadThroughWithExpirePolicy() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + Ignite ig = testedGrid(); CacheConfiguration cc = variationConfig("expire"); @@ -97,6 +107,8 @@ public void testReadThroughWithExpirePolicy() throws Exception { * @throws Exception if failed. */ public void testReadThroughExpirePolicyConfigured() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + Ignite ig = testedGrid(); CacheConfiguration cc = variationConfig("expireConfig"); @@ -150,6 +162,8 @@ public void testReadThroughExpirePolicyConfigured() throws Exception { * @throws Exception if failed. */ public void testReadThroughEvictionPolicy() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + Ignite ig = testedGrid(); CacheConfiguration cc = variationConfig("eviction"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionsVariationsSuite.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionsVariationsSuite.java index 400b2e95eee65..a390e121b9e50 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionsVariationsSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughEvictionsVariationsSuite.java @@ -29,9 +29,8 @@ public class IgniteCacheReadThroughEvictionsVariationsSuite extends TestSuite { /** * @return Cache API test suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { return new ConfigVariationsTestSuiteBuilder( "Cache Read Through Variations Test", IgniteCacheReadThroughEvictionSelfTest.class) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStoreCollectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStoreCollectionTest.java index 13841e3f3556d..4210822c408db 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStoreCollectionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStoreCollectionTest.java @@ -32,6 +32,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*; /** @@ -41,25 +42,34 @@ public class IgniteCacheStoreCollectionTest extends GridCommonAbstractTest { /** */ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + /** */ + private static final String CACHE1 = "cache1"; + + /** */ + private static final String CACHE2 = "cache2"; + + /** */ + private static final String CACHE3 = "cache3"; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder); - CacheConfiguration ccfg1 = new CacheConfiguration<>(DEFAULT_CACHE_NAME); - ccfg1.setName("cache1"); + CacheConfiguration ccfg1 = new CacheConfiguration<>(CACHE1); ccfg1.setAtomicityMode(ATOMIC); ccfg1.setWriteSynchronizationMode(FULL_SYNC); - CacheConfiguration ccfg2 = new CacheConfiguration<>(DEFAULT_CACHE_NAME); - ccfg2.setName("cache2"); + CacheConfiguration ccfg2 = new CacheConfiguration<>(CACHE2); ccfg2.setAtomicityMode(TRANSACTIONAL); ccfg2.setWriteSynchronizationMode(FULL_SYNC); - cfg.setCacheConfiguration(ccfg1, ccfg2); + CacheConfiguration ccfg3 = new CacheConfiguration<>(CACHE3); + ccfg3.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + ccfg3.setWriteSynchronizationMode(FULL_SYNC); - cfg.setMarshaller(null); + cfg.setCacheConfiguration(ccfg1, ccfg2, ccfg3); return cfg; } @@ -75,11 +85,13 @@ public class IgniteCacheStoreCollectionTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testStoreMap() throws Exception { - IgniteCache cache1 = ignite(0).cache("cache1"); - IgniteCache cache2 = ignite(0).cache("cache2"); + IgniteCache cache1 = ignite(0).cache(CACHE1); + IgniteCache cache2 = ignite(0).cache(CACHE2); + IgniteCache cache3 = ignite(0).cache(CACHE3); checkStoreMap(cache1); checkStoreMap(cache2); + checkStoreMap(cache3); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java index c777abaf4caeb..4ddb17d9bf06b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java @@ -93,6 +93,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -2048,6 +2049,9 @@ public void testServiceReassign() throws Exception { * @throws Exception If failed. */ public void testNoForceKeysRequests() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + cacheC = new IgniteClosure() { @Override public CacheConfiguration[] apply(String s) { return null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java index e09cf53c51c11..c3fa4f047eb08 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java @@ -62,23 +62,33 @@ public class NotMappedPartitionInTxTest extends GridCommonAbstractTest { /** Is client. */ private boolean isClient; + /** Atomicity mode. */ + private CacheAtomicityMode atomicityMode = CacheAtomicityMode.TRANSACTIONAL; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + atomicityMode = CacheAtomicityMode.TRANSACTIONAL; + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { return super.getConfiguration(gridName) .setClientMode(isClient) .setCacheConfiguration( new CacheConfiguration(CACHE) - .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setAtomicityMode(atomicityMode) .setCacheMode(CacheMode.REPLICATED) .setAffinity(new TestAffinity()), new CacheConfiguration(CACHE2) - .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + .setAtomicityMode(atomicityMode)); } /** * */ - public void testOneServerOptimistic() throws Exception { + public void testOneServerTx() throws Exception { try { isClient = false; startGrid(0); @@ -86,37 +96,11 @@ public void testOneServerOptimistic() throws Exception { isClient = true; final IgniteEx client = startGrid(1); - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - @Override public Void call() throws Exception { - testNotMapped(client, OPTIMISTIC, REPEATABLE_READ); - - return null; - } - }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); - } - finally { - stopAllGrids(); - } - } - - /** - * - */ - public void testOneServerOptimisticSerializable() throws Exception { - try { - isClient = false; - startGrid(0); + checkNotMapped(client, OPTIMISTIC, REPEATABLE_READ); - isClient = true; - final IgniteEx client = startGrid(1); - - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - @Override public Void call() throws Exception { - testNotMapped(client, OPTIMISTIC, SERIALIZABLE); + checkNotMapped(client, OPTIMISTIC, SERIALIZABLE); - return null; - } - }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + checkNotMapped(client, PESSIMISTIC, READ_COMMITTED); } finally { stopAllGrids(); @@ -126,21 +110,19 @@ public void testOneServerOptimisticSerializable() throws Exception { /** * */ - public void testOneServerPessimistic() throws Exception { + public void testOneServerMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10377"); + try { + atomicityMode = CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + isClient = false; startGrid(0); isClient = true; final IgniteEx client = startGrid(1); - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - @Override public Void call() throws Exception { - testNotMapped(client, PESSIMISTIC, READ_COMMITTED); - - return null; - } - }, ClusterTopologyServerNotFoundException.class, "Failed to lock keys (all partition nodes left the grid)"); + checkNotMapped(client, PESSIMISTIC, REPEATABLE_READ); } finally { stopAllGrids(); @@ -150,21 +132,19 @@ public void testOneServerPessimistic() throws Exception { /** * */ - public void testFourServersOptimistic() throws Exception { + public void testFourServersTx() throws Exception { try { isClient = false; - startGrids(4); + startGridsMultiThreaded(4); isClient = true; final IgniteEx client = startGrid(4); - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - @Override public Void call() throws Exception { - testNotMapped(client, OPTIMISTIC, REPEATABLE_READ); + checkNotMapped(client, OPTIMISTIC, REPEATABLE_READ); - return null; - } - }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + checkNotMapped(client, OPTIMISTIC, SERIALIZABLE); + + checkNotMapped(client, PESSIMISTIC, READ_COMMITTED); } finally { stopAllGrids(); @@ -174,21 +154,19 @@ public void testFourServersOptimistic() throws Exception { /** * */ - public void testFourServersOptimisticSerializable() throws Exception { + public void testFourServersMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10377"); + try { + atomicityMode = CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + isClient = false; - startGrids(4); + startGridsMultiThreaded(4); isClient = true; final IgniteEx client = startGrid(4); - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - @Override public Void call() throws Exception { - testNotMapped(client, OPTIMISTIC, SERIALIZABLE); - - return null; - } - }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + checkNotMapped(client, PESSIMISTIC, READ_COMMITTED); } finally { stopAllGrids(); @@ -196,50 +174,37 @@ public void testFourServersOptimisticSerializable() throws Exception { } /** - * + * @param client Ignite client. */ - public void testFourServersPessimistic() throws Exception { - try { - isClient = false; - startGrids(4); + private void checkNotMapped(final IgniteEx client, final TransactionConcurrency concurrency, + final TransactionIsolation isolation) { + String msg = concurrency == PESSIMISTIC ? "Failed to lock keys (all partition nodes left the grid)" : + "Failed to map keys to nodes (partition is not mapped to any node"; - isClient = true; - final IgniteEx client = startGrid(4); - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - @Override public Void call() throws Exception { - testNotMapped(client, PESSIMISTIC, READ_COMMITTED); + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() { + IgniteCache cache2 = client.cache(CACHE2); + IgniteCache cache1 = client.cache(CACHE).withKeepBinary(); - return null; - } - }, ClusterTopologyServerNotFoundException.class, "Failed to lock keys (all partition nodes left the grid)"); - } - finally { - stopAllGrids(); - } - } + try (Transaction tx = client.transactions().txStart(concurrency, isolation)) { - /** - * @param client Ignite client. - */ - private void testNotMapped(IgniteEx client, TransactionConcurrency concurrency, TransactionIsolation isolation) { - IgniteCache cache2 = client.cache(CACHE2); - IgniteCache cache1 = client.cache(CACHE).withKeepBinary(); + Map param = new TreeMap<>(); + param.put(TEST_KEY + 1, 1); + param.put(TEST_KEY + 1, 3); + param.put(TEST_KEY, 3); - try(Transaction tx = client.transactions().txStart(concurrency, isolation)) { + cache1.put(TEST_KEY, 3); - Map param = new TreeMap<>(); - param.put(TEST_KEY + 1, 1); - param.put(TEST_KEY + 1, 3); - param.put(TEST_KEY, 3); + cache1.putAll(param); + cache2.putAll(param); - cache1.put(TEST_KEY, 3); - - cache1.putAll(param); - cache2.putAll(param); + tx.commit(); + } - tx.commit(); - } + return null; + } + }, ClusterTopologyServerNotFoundException.class, msg); } /** */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java index 3a6ad48e3057b..eb090a8e9260d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/CacheManualRebalancingTest.java @@ -24,7 +24,6 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; -import org.apache.ignite.compute.ComputeTaskFuture; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; @@ -104,11 +103,9 @@ public void testRebalance() throws Exception { int newNodeCacheSize; // Start manual rebalancing. - IgniteCompute compute = newNode.compute().withAsync(); + IgniteCompute compute = newNode.compute(); - compute.broadcast(new MyCallable()); - - final ComputeTaskFuture rebalanceTaskFuture = compute.future(); + final IgniteFuture rebalanceTaskFuture = compute.broadcastAsync(new MyCallable()); boolean rebalanceFinished = GridTestUtils.waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java index e0d1eef95c18e..581865309fc69 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java @@ -110,12 +110,11 @@ public static TestSuite suite() { ignoredTests.add(NearCacheSyncUpdateTest.class); ignoredTests.add(GridCacheNearMultiGetSelfTest.class); - // Optimistic tx tests. + // Irrelevant Tx tests. ignoredTests.add(GridCacheColocatedOptimisticTransactionSelfTest.class); ignoredTests.add(CacheOptimisticTransactionsWithFilterSingleServerTest.class); ignoredTests.add(CacheOptimisticTransactionsWithFilterTest.class); - // Irrelevant Tx tests. ignoredTests.add(IgniteOnePhaseCommitInvokeTest.class); ignoredTests.add(IgniteOnePhaseCommitNearReadersTest.class); ignoredTests.add(GridCacheDhtPreloadOnheapSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java new file mode 100644 index 0000000000000..d422fc165a6da --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite5.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.GridCacheAffinityBackupsSelfTest; +import org.apache.ignite.IgniteCacheAffinitySelfTest; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.affinity.AffinityClientNodeSelfTest; +import org.apache.ignite.cache.affinity.AffinityDistributionLoggingTest; +import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest; +import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest; +import org.apache.ignite.internal.GridCachePartitionExchangeManagerHistSizeTest; +import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest; +import org.apache.ignite.internal.processors.cache.ClusterReadOnlyModeTest; +import org.apache.ignite.internal.processors.cache.ClusterStatePartitionedSelfTest; +import org.apache.ignite.internal.processors.cache.ClusterStateReplicatedSelfTest; +import org.apache.ignite.internal.processors.cache.ConcurrentCacheStartTest; +import org.apache.ignite.internal.processors.cache.EntryVersionConsistencyReadThroughTest; +import org.apache.ignite.internal.processors.cache.IgniteCachePutStackOverflowSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest; +import org.apache.ignite.internal.processors.cache.PartitionsExchangeOnDiscoveryHistoryOverflowTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.NotMappedPartitionInTxTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.IgniteCacheAtomicProtocolTest; +import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest; +import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest; + +/** + * Test suite. + */ +public class IgniteCacheMvccTestSuite5 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + HashSet ignoredTests = new HashSet<>(128); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(IgniteCacheStoreCollectionTest.class); + ignoredTests.add(EntryVersionConsistencyReadThroughTest.class); + ignoredTests.add(ClusterReadOnlyModeTest.class); + ignoredTests.add(NotMappedPartitionInTxTest.class); + ignoredTests.add(IgniteCacheTxIteratorSelfTest.class); + + // Irrelevant Tx tests. + ignoredTests.add(CacheSerializableTransactionsTest.class); + ignoredTests.add(IgniteCachePutStackOverflowSelfTest.class); + ignoredTests.add(IgniteCacheAtomicProtocolTest.class); + + // Other non-tx tests. + ignoredTests.add(CacheLateAffinityAssignmentNodeJoinValidationTest.class); + ignoredTests.add(IgniteCacheWriteBehindNoUpdateSelfTest.class); + ignoredTests.add(IgniteCacheSyncRebalanceModeSelfTest.class); + ignoredTests.add(ClusterStatePartitionedSelfTest.class); + ignoredTests.add(ClusterStateReplicatedSelfTest.class); + ignoredTests.add(CacheManualRebalancingTest.class); + ignoredTests.add(GridCacheAffinityBackupsSelfTest.class); + ignoredTests.add(IgniteCacheAffinitySelfTest.class); + ignoredTests.add(AffinityClientNodeSelfTest.class); + ignoredTests.add(LocalAffinityFunctionTest.class); + ignoredTests.add(AffinityHistoryCleanupTest.class); + ignoredTests.add(AffinityDistributionLoggingTest.class); + ignoredTests.add(PartitionsExchangeOnDiscoveryHistoryOverflowTest.class); + ignoredTests.add(GridCachePartitionExchangeManagerHistSizeTest.class); + ignoredTests.add(ConcurrentCacheStartTest.class); + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 5"); + + suite.addTest(IgniteCacheTestSuite5.suite(ignoredTests)); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index 8776901ebf75f..7bbc4ab0ee8d2 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.HashSet; import junit.framework.TestSuite; import org.apache.ignite.GridCacheAffinityBackupsSelfTest; import org.apache.ignite.IgniteCacheAffinitySelfTest; @@ -48,6 +49,7 @@ import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest; import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -55,57 +57,66 @@ public class IgniteCacheTestSuite5 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(HashSet ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 5"); - suite.addTestSuite(CacheSerializableTransactionsTest.class); - suite.addTestSuite(CacheNearReaderUpdateTest.class); - suite.addTestSuite(IgniteCacheStoreCollectionTest.class); - suite.addTestSuite(IgniteCacheWriteBehindNoUpdateSelfTest.class); - suite.addTestSuite(IgniteCachePutStackOverflowSelfTest.class); - suite.addTestSuite(CacheKeepBinaryTransactionTest.class); + GridTestUtils.addTestIfNeeded(suite,CacheSerializableTransactionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,CacheNearReaderUpdateTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheStoreCollectionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheWriteBehindNoUpdateSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCachePutStackOverflowSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,CacheKeepBinaryTransactionTest.class, ignoredTests); - suite.addTestSuite(CacheLateAffinityAssignmentTest.class); - suite.addTestSuite(CacheLateAffinityAssignmentNodeJoinValidationTest.class); - suite.addTestSuite(EntryVersionConsistencyReadThroughTest.class); - suite.addTestSuite(IgniteCacheSyncRebalanceModeSelfTest.class); + GridTestUtils.addTestIfNeeded(suite,CacheLateAffinityAssignmentTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,CacheLateAffinityAssignmentNodeJoinValidationTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,EntryVersionConsistencyReadThroughTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheSyncRebalanceModeSelfTest.class, ignoredTests); suite.addTest(IgniteCacheReadThroughEvictionsVariationsSuite.suite()); - suite.addTestSuite(IgniteCacheTxIteratorSelfTest.class); - suite.addTestSuite(ClusterStatePartitionedSelfTest.class); - suite.addTestSuite(ClusterStateReplicatedSelfTest.class); - suite.addTestSuite(ClusterReadOnlyModeTest.class); - suite.addTestSuite(IgniteCachePartitionLossPolicySelfTest.class); - suite.addTestSuite(IgniteCacheGroupsPartitionLossPolicySelfTest.class); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheTxIteratorSelfTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite,ClusterStatePartitionedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,ClusterStateReplicatedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,ClusterReadOnlyModeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCachePartitionLossPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheGroupsPartitionLossPolicySelfTest.class, ignoredTests); - suite.addTestSuite(CacheRebalancingSelfTest.class); - suite.addTestSuite(CacheManualRebalancingTest.class); + GridTestUtils.addTestIfNeeded(suite,CacheRebalancingSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,CacheManualRebalancingTest.class, ignoredTests); // Affinity tests. - suite.addTestSuite(GridCacheAffinityBackupsSelfTest.class); - suite.addTestSuite(IgniteCacheAffinitySelfTest.class); - suite.addTestSuite(AffinityClientNodeSelfTest.class); - suite.addTestSuite(LocalAffinityFunctionTest.class); - suite.addTestSuite(AffinityHistoryCleanupTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCacheAffinityBackupsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheAffinitySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,AffinityClientNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,LocalAffinityFunctionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite,AffinityHistoryCleanupTest.class, ignoredTests); + + GridTestUtils.addTestIfNeeded(suite,AffinityDistributionLoggingTest.class, ignoredTests); - suite.addTestSuite(AffinityDistributionLoggingTest.class); + GridTestUtils.addTestIfNeeded(suite,IgniteCacheAtomicProtocolTest.class, ignoredTests); - suite.addTestSuite(IgniteCacheAtomicProtocolTest.class); + GridTestUtils.addTestIfNeeded(suite,PartitionsExchangeOnDiscoveryHistoryOverflowTest.class, ignoredTests); - suite.addTestSuite(PartitionsExchangeOnDiscoveryHistoryOverflowTest.class); + GridTestUtils.addTestIfNeeded(suite,GridCachePartitionExchangeManagerHistSizeTest.class, ignoredTests); - suite.addTestSuite(GridCachePartitionExchangeManagerHistSizeTest.class); + GridTestUtils.addTestIfNeeded(suite,NotMappedPartitionInTxTest.class, ignoredTests); - suite.addTestSuite(NotMappedPartitionInTxTest.class); + GridTestUtils.addTestIfNeeded(suite,ConcurrentCacheStartTest.class, ignoredTests); - suite.addTestSuite(ConcurrentCacheStartTest.class); + //GridTestUtils.addTestIfNeeded(suite,GridCacheAtomicPreloadSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheContainsKeyColocatedAtomicSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheContainsKeyNearAtomicSelfTest.class, ignoredTests); - //suite.addTestSuite(GridCacheAtomicPreloadSelfTest.class); - //suite.addTestSuite(IgniteCacheContainsKeyColocatedAtomicSelfTest.class); - //suite.addTestSuite(IgniteCacheContainsKeyNearAtomicSelfTest.class); return suite; } } From 1f2a817c81ce20bfaf9656979f29853d446ff485 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Fri, 30 Nov 2018 16:06:03 +0300 Subject: [PATCH 193/403] IGNITE-10365: MVCC: Create "Cache 6" test suite for MVCC mode. This closes #5478. (cherry picked from commit 74d342d666bbe593b98224f0dc73d94bad885a32) --- .../CacheGetsDistributionAbstractTest.java | 375 ++++++++++++++++++ .../cache/CacheNoAffinityExchangeTest.java | 8 + ...tionedAtomicCacheGetsDistributionTest.java | 24 +- ...xPessimisticCacheGetsDistributionTest.java | 32 ++ ...alOptimisticCacheGetsDistributionTest.java | 17 +- ...lPessimisticCacheGetsDistributionTest.java | 24 +- ...icatedAtomicCacheGetsDistributionTest.java | 329 +-------------- ...xPessimisticCacheGetsDistributionTest.java | 32 ++ ...alOptimisticCacheGetsDistributionTest.java | 9 +- ...lPessimisticCacheGetsDistributionTest.java | 24 +- .../distributed/CacheExchangeMergeTest.java | 9 +- .../CacheTryLockMultithreadedTest.java | 3 + .../IgniteCache150ClientsTest.java | 10 +- .../IgniteCacheThreadLocalTxTest.java | 4 + .../IgnitePessimisticTxSuspendResumeTest.java | 5 + .../cache/transactions/TxLabelTest.java | 23 +- .../transactions/TxRollbackAsyncTest.java | 44 +- .../TxRollbackOnIncorrectParamsTest.java | 17 +- .../transactions/TxRollbackOnTimeoutTest.java | 22 +- .../transactions/TxStateChangeEventTest.java | 78 +++- .../testframework/MvccFeatureChecker.java | 4 + .../testsuites/IgniteCacheMvccTestSuite6.java | 94 +++++ .../testsuites/IgniteCacheTestSuite6.java | 85 ++-- 23 files changed, 810 insertions(+), 462 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetsDistributionAbstractTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedMvccTxPessimisticCacheGetsDistributionTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedMvccTxPessimisticCacheGetsDistributionTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetsDistributionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetsDistributionAbstractTest.java new file mode 100644 index 0000000000000..559a719c55080 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetsDistributionAbstractTest.java @@ -0,0 +1,375 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests of replicated cache's 'get' requests distribution. + */ +public abstract class CacheGetsDistributionAbstractTest extends GridCommonAbstractTest { + /** Client nodes instance's name. */ + private static final String CLIENT_NAME = "client"; + + /** Value prefix. */ + private static final String VAL_PREFIX = "val"; + + /** */ + private static final int PRIMARY_KEYS_NUMBER = 1_000; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + assert gridCount() >= 1 : "At least one grid must be started"; + + startGridsMultiThreaded(gridCount()); + + IgniteConfiguration clientCfg = getConfiguration(CLIENT_NAME); + + clientCfg.setClientMode(true); + + startGrid(clientCfg); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + IgniteCache cache = ignite(0).cache(DEFAULT_CACHE_NAME); + + if (cache != null) + cache.destroy(); + + // Setting different MAC addresses for all nodes + Map macs = getClusterMacs(); + + int idx = 0; + + for (Map.Entry entry : macs.entrySet()) + entry.setValue("x2-xx-xx-xx-xx-x" + idx++); + + replaceMacAddresses(G.allGrids(), macs); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + TransactionConfiguration txCfg = new TransactionConfiguration() + .setDefaultTxIsolation(transactionIsolation()) + .setDefaultTxConcurrency(transactionConcurrency()); + + cfg.setTransactionConfiguration(txCfg); + + return cfg; + } + + /** + * @return Grids count to start. + */ + protected int gridCount() { + return 4; + } + + /** + * @return Cache configuration. + */ + protected CacheConfiguration cacheConfiguration() { + CacheConfiguration ccfg = defaultCacheConfiguration(); + + ccfg.setCacheMode(cacheMode()); + ccfg.setAtomicityMode(atomicityMode()); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + ccfg.setReadFromBackup(true); + ccfg.setStatisticsEnabled(true); + + if (cacheMode() == CacheMode.PARTITIONED) + ccfg.setBackups(backupsCount()); + + return ccfg; + } + + /** + * @return Cache mode. + */ + protected abstract CacheMode cacheMode(); + + /** + * @return Cache atomicity mode. + */ + protected abstract CacheAtomicityMode atomicityMode(); + + /** + * @return Cache transaction isolation. + */ + protected TransactionIsolation transactionIsolation() { + return REPEATABLE_READ; + } + + /** + * @return Cache transaction concurrency. + */ + protected TransactionConcurrency transactionConcurrency() { + return PESSIMISTIC; + } + + /** + * @return Backups count. + */ + protected int backupsCount() { + return gridCount() - 1; + } + + /** + * Test 'get' operations requests generator distribution. + * + * @throws Exception In case of an error. + * @see #runTestBalancingDistribution(boolean) + */ + public void testGetRequestsGeneratorDistribution() throws Exception { + runTestBalancingDistribution(false); + } + + /** + * Test 'getAll' operations requests generator distribution. + * + * @throws Exception In case of an error. + * @see #runTestBalancingDistribution(boolean) + */ + public void testGetAllRequestsGeneratorDistribution() throws Exception { + runTestBalancingDistribution(true); + } + + /** + * @param batchMode Whenever 'get' or 'getAll' operations are used in the test. + * @throws Exception In case of an error. + */ + protected void runTestBalancingDistribution(boolean batchMode) throws Exception { + IgniteCache cache = grid(0).createCache(cacheConfiguration()); + + List keys = primaryKeys(cache, PRIMARY_KEYS_NUMBER); + + for (Integer key : keys) + cache.put(key, VAL_PREFIX + key); + + IgniteCache clientCache = grid(CLIENT_NAME).cache(DEFAULT_CACHE_NAME) + .withAllowAtomicOpsInTx(); + + assertTrue(GridTestUtils.waitForCondition( + new GridAbsPredicate() { + int batchSize = 10; + int idx = 0; + + @Override public boolean apply() { + if (idx >= PRIMARY_KEYS_NUMBER) + idx = 0; + + try (Transaction tx = grid(CLIENT_NAME).transactions().txStart()) { + if (batchMode) { + Set keys0 = new TreeSet<>(); + + for (int i = idx; i < idx + batchSize && i < PRIMARY_KEYS_NUMBER; i++) + keys0.add(keys.get(i)); + + idx += batchSize; + + Map results = clientCache.getAll(keys0); + + for (Map.Entry entry : results.entrySet()) + assertEquals(VAL_PREFIX + entry.getKey(), entry.getValue()); + } + else { + for (int i = idx; i < idx + gridCount() && i < PRIMARY_KEYS_NUMBER; i++) { + Integer key = keys.get(i); + + assertEquals(VAL_PREFIX + key, clientCache.get(key)); + } + + idx += gridCount(); + } + + tx.commit(); + } + + for (int i = 0; i < gridCount(); i++) { + IgniteEx ignite = grid(i); + + long getsCnt = ignite.cache(DEFAULT_CACHE_NAME).localMetrics().getCacheGets(); + + if (getsCnt == 0) + return false; + } + + return true; + } + }, + getTestTimeout()) + ); + } + + /** + * Tests that the 'get' operation requests are routed to node with same MAC address as at requester. + * + * @throws Exception In case of an error. + * @see #runTestSameHostDistribution(UUID, boolean) + */ + public void testGetRequestsDistribution() throws Exception { + UUID destId = grid(0).localNode().id(); + + runTestSameHostDistribution(destId, false); + } + + /** + * Tests that the 'getAll' operation requests are routed to node with same MAC address as at requester. + * + * @throws Exception In case of an error. + * @see #runTestSameHostDistribution(UUID, boolean) + */ + public void testGetAllRequestsDistribution() throws Exception { + UUID destId = grid(gridCount() - 1).localNode().id(); + + runTestSameHostDistribution(destId, true); + } + + /** + * Tests that the 'get' and 'getAll' requests are routed to node with same MAC address as at requester. + * + * @param destId Destination Ignite instance id for requests distribution. + * @param batchMode Test mode. + * @throws Exception In case of an error. + */ + protected void runTestSameHostDistribution(final UUID destId, final boolean batchMode) throws Exception { + Map macs = getClusterMacs(); + + String clientMac = macs.get(grid(CLIENT_NAME).localNode().id()); + + macs.put(destId, clientMac); + + replaceMacAddresses(G.allGrids(), macs); + + IgniteCache cache = grid(0).createCache(cacheConfiguration()); + + List keys = primaryKeys(cache, PRIMARY_KEYS_NUMBER); + + for (Integer key : keys) + cache.put(key, VAL_PREFIX + key); + + IgniteCache clientCache = grid(CLIENT_NAME).cache(DEFAULT_CACHE_NAME) + .withAllowAtomicOpsInTx(); + + try (Transaction tx = grid(CLIENT_NAME).transactions().txStart()) { + if (batchMode) { + Map results = clientCache.getAll(new TreeSet<>(keys)); + + for (Map.Entry entry : results.entrySet()) + assertEquals(VAL_PREFIX + entry.getKey(), entry.getValue()); + } + else { + for (Integer key : keys) + assertEquals(VAL_PREFIX + key, clientCache.get(key)); + } + + tx.commit(); + } + + for (int i = 0; i < gridCount(); i++) { + IgniteEx ignite = grid(i); + + long getsCnt = ignite.cache(DEFAULT_CACHE_NAME).localMetrics().getCacheGets(); + + if (destId.equals(ignite.localNode().id())) + assertEquals(PRIMARY_KEYS_NUMBER, getsCnt); + else + assertEquals(0L, getsCnt); + } + } + + /** + * @param instances Started Ignite instances. + * @param macs Mapping MAC addresses to UUID. + */ + private void replaceMacAddresses(List instances, Map macs) { + for (Ignite ignite : instances) { + for (ClusterNode node : ignite.cluster().nodes()) { + String mac = macs.get(node.id()); + + assertNotNull(mac); + + Map attrs = new HashMap<>(node.attributes()); + + attrs.put(ATTR_MACS, mac); + + ((TcpDiscoveryNode)node).setAttributes(attrs); + } + } + } + + /** + * @return Cluster nodes MAC addresses. + */ + private Map getClusterMacs() { + Map macs = new HashMap<>(); + + for (Ignite ignite : G.allGrids()) { + ClusterNode node = ignite.cluster().localNode(); + + String mac = node.attribute(ATTR_MACS); + + assert mac != null; + + macs.put(node.id(), mac); + } + + return macs; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java index 45c6f25bf275a..15cd5eeb0a14d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage; import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -51,6 +52,13 @@ public class CacheNoAffinityExchangeTest extends GridCommonAbstractTest { private final TcpDiscoveryIpFinder CLIENT_IP_FINDER = new TcpDiscoveryVmIpFinder() .setAddresses(Collections.singleton("127.0.0.1:47500")); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedAtomicCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedAtomicCacheGetsDistributionTest.java index 2241a956e0908..963ccc81d02c8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedAtomicCacheGetsDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedAtomicCacheGetsDistributionTest.java @@ -17,33 +17,23 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.configuration.CacheConfiguration; +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheMode.PARTITIONED; /** * Tests of partitioned atomic cache's 'get' requests distribution. */ -public class PartitionedAtomicCacheGetsDistributionTest extends ReplicatedAtomicCacheGetsDistributionTest { +public class PartitionedAtomicCacheGetsDistributionTest extends CacheGetsDistributionAbstractTest { /** {@inheritDoc} */ - @Override protected CacheMode cacheMode() { - return PARTITIONED; + @Override protected CacheAtomicityMode atomicityMode() { + return ATOMIC; } /** {@inheritDoc} */ - @Override protected CacheConfiguration cacheConfiguration() { - CacheConfiguration cacheCfg = super.cacheConfiguration(); - - cacheCfg.setBackups(backupsCount()); - - return cacheCfg; - } - - /** - * @return Backups count. - */ - protected int backupsCount() { - return gridCount() - 1; + @Override protected CacheMode cacheMode() { + return PARTITIONED; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedMvccTxPessimisticCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedMvccTxPessimisticCacheGetsDistributionTest.java new file mode 100644 index 0000000000000..de9d9a658afc6 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedMvccTxPessimisticCacheGetsDistributionTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests of pessimistic transactional partitioned cache's 'get' requests distribution. + */ +public class PartitionedMvccTxPessimisticCacheGetsDistributionTest extends PartitionedTransactionalPessimisticCacheGetsDistributionTest { + /** {@inheritDoc} */ + @Override protected TransactionIsolation transactionIsolation() { + return REPEATABLE_READ; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalOptimisticCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalOptimisticCacheGetsDistributionTest.java index 4c882294f89eb..e518e32ef5340 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalOptimisticCacheGetsDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalOptimisticCacheGetsDistributionTest.java @@ -18,29 +18,36 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; /** * Tests of optimistic transactional partitioned cache's 'get' requests distribution. */ -public class PartitionedTransactionalOptimisticCacheGetsDistributionTest extends PartitionedAtomicCacheGetsDistributionTest { +public class PartitionedTransactionalOptimisticCacheGetsDistributionTest extends CacheGetsDistributionAbstractTest { /** {@inheritDoc} */ - @Override protected CacheAtomicityMode atomicityMode() { - return TRANSACTIONAL; + @Override protected CacheMode cacheMode() { + return PARTITIONED; } /** {@inheritDoc} */ - @Override protected TransactionIsolation transactionIsolation() { - return READ_COMMITTED; + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL; } /** {@inheritDoc} */ @Override protected TransactionConcurrency transactionConcurrency() { return OPTIMISTIC; } + + /** {@inheritDoc} */ + @Override protected TransactionIsolation transactionIsolation() { + return READ_COMMITTED; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalPessimisticCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalPessimisticCacheGetsDistributionTest.java index 78ea7a6f8d30d..ce1f7b66ceaa5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalPessimisticCacheGetsDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionedTransactionalPessimisticCacheGetsDistributionTest.java @@ -17,17 +17,37 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; /** * Tests of pessimistic transactional partitioned cache's 'get' requests distribution. */ -public class PartitionedTransactionalPessimisticCacheGetsDistributionTest - extends PartitionedTransactionalOptimisticCacheGetsDistributionTest { +public class PartitionedTransactionalPessimisticCacheGetsDistributionTest extends CacheGetsDistributionAbstractTest { + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return PARTITIONED; + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL; + } + /** {@inheritDoc} */ @Override protected TransactionConcurrency transactionConcurrency() { return PESSIMISTIC; } + + /** {@inheritDoc} */ + @Override protected TransactionIsolation transactionIsolation() { + return READ_COMMITTED; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedAtomicCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedAtomicCacheGetsDistributionTest.java index 1aaea761d084a..ef90408e4a20a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedAtomicCacheGetsDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedAtomicCacheGetsDistributionTest.java @@ -17,350 +17,23 @@ package org.apache.ignite.internal.processors.cache; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.UUID; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.TransactionConfiguration; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.util.lang.GridAbsPredicate; -import org.apache.ignite.internal.util.typedef.G; -import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode; -import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.transactions.Transaction; -import org.apache.ignite.transactions.TransactionConcurrency; -import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheMode.REPLICATED; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; -import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; -import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; -import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; /** * Tests of replicated cache's 'get' requests distribution. */ -public class ReplicatedAtomicCacheGetsDistributionTest extends GridCacheAbstractSelfTest { - /** Cache name. */ - private static final String CACHE_NAME = "getsDistributionTest"; - - /** Client nodes instance's name. */ - private static final String CLIENT_NAME = "client"; - - /** Value prefix. */ - private static final String VAL_PREFIX = "val"; - - /** */ - private static final int PRIMARY_KEYS_NUMBER = 1_000; - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - - IgniteConfiguration clientCfg = getConfiguration(CLIENT_NAME); - - clientCfg.setClientMode(true); - - startGrid(clientCfg); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - IgniteCache cache = ignite(0).cache(CACHE_NAME); - - if (cache != null) - cache.destroy(); - - // Setting different MAC addresses for all nodes - Map macs = getClusterMacs(); - - int idx = 0; - - for (Map.Entry entry : macs.entrySet()) - entry.setValue("x2-xx-xx-xx-xx-x" + idx++); - - replaceMacAddresses(G.allGrids(), macs); - } - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setTransactionConfiguration(transactionConfiguration()); - - return cfg; - } - +public class ReplicatedAtomicCacheGetsDistributionTest extends CacheGetsDistributionAbstractTest { /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return ATOMIC; } /** {@inheritDoc} */ - @Override protected int gridCount() { - return 4; - } - - /** - * Test 'get' operations requests generator distribution. - * - * @throws Exception In case of an error. - * @see #runTestBalancingDistribution(boolean) - */ - public void testGetRequestsGeneratorDistribution() throws Exception { - runTestBalancingDistribution(false); - } - - /** - * Test 'getAll' operations requests generator distribution. - * - * @throws Exception In case of an error. - * @see #runTestBalancingDistribution(boolean) - */ - public void testGetAllRequestsGeneratorDistribution() throws Exception { - runTestBalancingDistribution(true); - } - - /** - * @param batchMode Whenever 'get' or 'getAll' operations are used in the test. - * @throws Exception In case of an error. - */ - protected void runTestBalancingDistribution(boolean batchMode) throws Exception { - IgniteCache cache = grid(0).createCache(cacheConfiguration()); - - List keys = primaryKeys(cache, PRIMARY_KEYS_NUMBER); - - for (Integer key : keys) - cache.put(key, VAL_PREFIX + key); - - IgniteCache clientCache = grid(CLIENT_NAME).getOrCreateCache(CACHE_NAME) - .withAllowAtomicOpsInTx(); - - assertTrue(GridTestUtils.waitForCondition( - new GridAbsPredicate() { - int batchSize = 10; - int idx = 0; - - @Override public boolean apply() { - if (idx >= PRIMARY_KEYS_NUMBER) - idx = 0; - - try (Transaction tx = grid(CLIENT_NAME).transactions().txStart()) { - if (batchMode) { - Set keys0 = new TreeSet<>(); - - for (int i = idx; i < idx + batchSize && i < PRIMARY_KEYS_NUMBER; i++) - keys0.add(keys.get(i)); - - idx += batchSize; - - Map results = clientCache.getAll(keys0); - - for (Map.Entry entry : results.entrySet()) - assertEquals(VAL_PREFIX + entry.getKey(), entry.getValue()); - } - else { - for (int i = idx; i < idx + gridCount() && i < PRIMARY_KEYS_NUMBER; i++) { - Integer key = keys.get(i); - - assertEquals(VAL_PREFIX + key, clientCache.get(key)); - } - - idx += gridCount(); - } - - tx.commit(); - } - - for (int i = 0; i < gridCount(); i++) { - IgniteEx ignite = grid(i); - - long getsCnt = ignite.cache(CACHE_NAME).localMetrics().getCacheGets(); - - if (getsCnt == 0) - return false; - } - - return true; - } - }, - getTestTimeout()) - ); - } - - /** - * Tests that the 'get' operation requests are routed to node with same MAC address as at requester. - * - * @throws Exception In case of an error. - * @see #runTestSameHostDistribution(UUID, boolean) - */ - public void testGetRequestsDistribution() throws Exception { - UUID destId = grid(0).localNode().id(); - - runTestSameHostDistribution(destId, false); - } - - /** - * Tests that the 'getAll' operation requests are routed to node with same MAC address as at requester. - * - * @throws Exception In case of an error. - * @see #runTestSameHostDistribution(UUID, boolean) - */ - public void testGetAllRequestsDistribution() throws Exception { - UUID destId = grid(gridCount() - 1).localNode().id(); - - runTestSameHostDistribution(destId, true); - } - - /** - * Tests that the 'get' and 'getAll' requests are routed to node with same MAC address as at requester. - * - * @param destId Destination Ignite instance id for requests distribution. - * @param batchMode Test mode. - * @throws Exception In case of an error. - */ - protected void runTestSameHostDistribution(final UUID destId, final boolean batchMode) throws Exception { - Map macs = getClusterMacs(); - - String clientMac = macs.get(grid(CLIENT_NAME).localNode().id()); - - macs.put(destId, clientMac); - - replaceMacAddresses(G.allGrids(), macs); - - IgniteCache cache = grid(0).createCache(cacheConfiguration()); - - List keys = primaryKeys(cache, PRIMARY_KEYS_NUMBER); - - for (Integer key : keys) - cache.put(key, VAL_PREFIX + key); - - IgniteCache clientCache = grid(CLIENT_NAME).getOrCreateCache(CACHE_NAME) - .withAllowAtomicOpsInTx(); - - try (Transaction tx = grid(CLIENT_NAME).transactions().txStart()) { - if (batchMode) { - Map results = clientCache.getAll(new TreeSet<>(keys)); - - for (Map.Entry entry : results.entrySet()) - assertEquals(VAL_PREFIX + entry.getKey(), entry.getValue()); - } - else { - for (Integer key : keys) - assertEquals(VAL_PREFIX + key, clientCache.get(key)); - } - - tx.commit(); - } - - for (int i = 0; i < gridCount(); i++) { - IgniteEx ignite = grid(i); - - long getsCnt = ignite.cache(CACHE_NAME).localMetrics().getCacheGets(); - - if (destId.equals(ignite.localNode().id())) - assertEquals(PRIMARY_KEYS_NUMBER, getsCnt); - else - assertEquals(0L, getsCnt); - } - } - - /** - * @return Transaction configuration. - */ - protected TransactionConfiguration transactionConfiguration() { - TransactionConfiguration txCfg = new TransactionConfiguration(); - - txCfg.setDefaultTxIsolation(transactionIsolation()); - txCfg.setDefaultTxConcurrency(transactionConcurrency()); - - return txCfg; - } - - /** - * @return Cache transaction isolation. - */ - protected TransactionIsolation transactionIsolation() { - return REPEATABLE_READ; - } - - /** - * @return Cache transaction concurrency. - */ - protected TransactionConcurrency transactionConcurrency() { - return PESSIMISTIC; - } - - /** - * @return Caching mode. - */ @Override protected CacheMode cacheMode() { return REPLICATED; } - - /** - * @return Cache configuration. - */ - protected CacheConfiguration cacheConfiguration() { - CacheConfiguration cfg = new CacheConfiguration(CACHE_NAME); - - cfg.setCacheMode(cacheMode()); - cfg.setAtomicityMode(atomicityMode()); - cfg.setWriteSynchronizationMode(FULL_SYNC); - cfg.setReadFromBackup(true); - cfg.setStatisticsEnabled(true); - - return cfg; - } - - /** - * @param instances Started Ignite instances. - * @param macs Mapping MAC addresses to UUID. - */ - private void replaceMacAddresses(List instances, Map macs) { - for (Ignite ignite : instances) { - for (ClusterNode node : ignite.cluster().nodes()) { - String mac = macs.get(node.id()); - - assertNotNull(mac); - - Map attrs = new HashMap<>(node.attributes()); - - attrs.put(ATTR_MACS, mac); - - ((TcpDiscoveryNode)node).setAttributes(attrs); - } - } - } - - /** - * @return Cluster nodes MAC addresses. - */ - private Map getClusterMacs() { - Map macs = new HashMap<>(); - - for (Ignite ignite : G.allGrids()) { - ClusterNode node = ignite.cluster().localNode(); - - String mac = node.attribute(ATTR_MACS); - - assert mac != null; - - macs.put(node.id(), mac); - } - - return macs; - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedMvccTxPessimisticCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedMvccTxPessimisticCacheGetsDistributionTest.java new file mode 100644 index 0000000000000..6e2d67c3137d7 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedMvccTxPessimisticCacheGetsDistributionTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Tests of pessimistic transactional replicated cache's 'get' requests distribution. + */ +public class ReplicatedMvccTxPessimisticCacheGetsDistributionTest extends ReplicatedTransactionalPessimisticCacheGetsDistributionTest { + /** {@inheritDoc} */ + @Override protected TransactionIsolation transactionIsolation() { + return REPEATABLE_READ; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalOptimisticCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalOptimisticCacheGetsDistributionTest.java index 3bc680972c050..4744f0a1eaba2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalOptimisticCacheGetsDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalOptimisticCacheGetsDistributionTest.java @@ -18,17 +18,24 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; /** * Tests of optimistic transactional replicated cache's 'get' requests distribution. */ -public class ReplicatedTransactionalOptimisticCacheGetsDistributionTest extends ReplicatedAtomicCacheGetsDistributionTest { +public class ReplicatedTransactionalOptimisticCacheGetsDistributionTest extends CacheGetsDistributionAbstractTest { + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return REPLICATED; + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalPessimisticCacheGetsDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalPessimisticCacheGetsDistributionTest.java index 7bace3c74fddc..2648851d426be 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalPessimisticCacheGetsDistributionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ReplicatedTransactionalPessimisticCacheGetsDistributionTest.java @@ -17,17 +17,37 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; /** * Tests of pessimistic transactional replicated cache's 'get' requests distribution. */ -public class ReplicatedTransactionalPessimisticCacheGetsDistributionTest - extends ReplicatedTransactionalOptimisticCacheGetsDistributionTest { +public class ReplicatedTransactionalPessimisticCacheGetsDistributionTest extends CacheGetsDistributionAbstractTest { + /** {@inheritDoc} */ + @Override protected CacheMode cacheMode() { + return REPLICATED; + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL; + } + /** {@inheritDoc} */ @Override protected TransactionConcurrency transactionConcurrency() { return PESSIMISTIC; } + + /** {@inheritDoc} */ + @Override protected TransactionIsolation transactionIsolation() { + return READ_COMMITTED; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java index 49318f9719b23..c5a708748f1d5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java @@ -79,6 +79,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -149,7 +150,12 @@ else if (testDelaySpi) cacheConfiguration("c7", TRANSACTIONAL, PARTITIONED, 1), cacheConfiguration("c8", TRANSACTIONAL, PARTITIONED, 2), cacheConfiguration("c9", TRANSACTIONAL, PARTITIONED, 10), - cacheConfiguration("c10", TRANSACTIONAL, REPLICATED, 0) + cacheConfiguration("c10", TRANSACTIONAL, REPLICATED, 0), + cacheConfiguration("c11", TRANSACTIONAL_SNAPSHOT, PARTITIONED, 0), + cacheConfiguration("c12", TRANSACTIONAL_SNAPSHOT, PARTITIONED, 1), + cacheConfiguration("c13", TRANSACTIONAL_SNAPSHOT, PARTITIONED, 2), + cacheConfiguration("c14", TRANSACTIONAL_SNAPSHOT, PARTITIONED, 10), + cacheConfiguration("c15", TRANSACTIONAL_SNAPSHOT, REPLICATED, 0) ); } @@ -823,7 +829,6 @@ public void testMergeServersAndClientsFail2() throws Exception { mergeServersAndClientsFail(true); } - /** * @param waitRebalance Wait for rebalance end before start tested topology change. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTryLockMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTryLockMultithreadedTest.java index 82d9c129bcebd..bccc703174e0d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTryLockMultithreadedTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTryLockMultithreadedTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -68,6 +69,8 @@ public class CacheTryLockMultithreadedTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + super.beforeTestsStarted(); startGridsMultiThreaded(SRVS); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java index b7ae84400c2fe..62305014ddaf6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.ClientConnectorConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -37,8 +38,6 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; @@ -52,6 +51,9 @@ public class IgniteCache150ClientsTest extends GridCommonAbstractTest { /** */ private static final int CACHES = 10; + /** */ + private static final int CLIENTS = 150; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -80,7 +82,7 @@ public class IgniteCache150ClientsTest extends GridCommonAbstractTest { CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); ccfg.setCacheMode(PARTITIONED); - ccfg.setAtomicityMode(i % 2 == 0 ? ATOMIC : TRANSACTIONAL); + ccfg.setAtomicityMode(CacheAtomicityMode.values()[i % 3]); ccfg.setWriteSynchronizationMode(PRIMARY_SYNC); ccfg.setBackups(1); @@ -114,8 +116,6 @@ public void test150Clients() throws Exception { assertFalse(srv.configuration().isClientMode()); - final int CLIENTS = 150; - final AtomicInteger idx = new AtomicInteger(1); final CountDownLatch latch = new CountDownLatch(CLIENTS); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java index c8eac20782fe5..cab6c720cca16 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -106,6 +107,9 @@ private void threadLocalTx(Ignite node) throws Exception { for (TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + for (boolean read : reads) { for (boolean write : writes) { for (int i = 0; i < endOps; i++) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgnitePessimisticTxSuspendResumeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgnitePessimisticTxSuspendResumeTest.java index 57a1470010578..dba769d2d560c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgnitePessimisticTxSuspendResumeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgnitePessimisticTxSuspendResumeTest.java @@ -24,6 +24,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -70,6 +71,10 @@ public void testSuspendPessimisticTx() throws Exception { IgniteTransactions txs = g.transactions(); for (TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && + !MvccFeatureChecker.isSupported(TransactionConcurrency.PESSIMISTIC, isolation)) + continue; + final Transaction tx = txs.txStart(TransactionConcurrency.PESSIMISTIC, isolation); cache.put(1, "1"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java index d89ba0b6554ec..50b107d042e6f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java @@ -18,13 +18,27 @@ package org.apache.ignite.internal.processors.cache.transactions; import org.apache.ignite.Ignite; -import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; /** * Tests transaction labels. */ -public class TxLabelTest extends GridCacheAbstractSelfTest { +public class TxLabelTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrid(0).getOrCreateCache(defaultCacheConfiguration()); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + /** * Tests transaction labels. */ @@ -55,9 +69,4 @@ private void testLabel0(Ignite ignite, String lbl) { tx.commit(); } } - - /** {@inheritDoc} */ - @Override protected int gridCount() { - return 1; - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java index 4ca8ba37c34ba..4a074e6f1225a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java @@ -81,6 +81,7 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -123,6 +124,14 @@ public class TxRollbackAsyncTest extends GridCommonAbstractTest { /** */ public static final String LABEL = "wLockTx"; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10410"); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -165,10 +174,11 @@ protected boolean nearCacheEnabled() { } /** - * * @return {@code True} if persistence must be enabled for test. */ - protected boolean persistenceEnabled() { return false; } + protected boolean persistenceEnabled() { + return false; + } /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { @@ -339,14 +349,13 @@ public void testSynchronousRollback() throws Exception { } /** - * * @param holdLockNode Node holding the write lock. * @param tryLockNode Node trying to acquire lock. * @param useTimeout {@code True} if need to start tx with timeout. - * * @throws Exception If failed. */ - private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockNode, final boolean useTimeout) throws Exception { + private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockNode, + final boolean useTimeout) throws Exception { final CountDownLatch keyLocked = new CountDownLatch(1); CountDownLatch waitCommit = new CountDownLatch(1); @@ -409,7 +418,7 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN int proc = 1; - while(true) { + while (true) { try { Transaction tx = txReadyFut.get(); @@ -490,11 +499,11 @@ public void testEnlistManyWriteOptimistic() throws Exception { testEnlistMany(true, SERIALIZABLE, OPTIMISTIC); } - /** * */ - private void testEnlistMany(boolean write, TransactionIsolation isolation, TransactionConcurrency conc) throws Exception { + private void testEnlistMany(boolean write, TransactionIsolation isolation, + TransactionConcurrency conc) throws Exception { final Ignite client = startClient(); Map entries = new HashMap<>(); @@ -504,7 +513,7 @@ private void testEnlistMany(boolean write, TransactionIsolation isolation, Trans IgniteInternalFuture fut = null; - try(Transaction tx = client.transactions().txStart(conc, isolation, 0, 0)) { + try (Transaction tx = client.transactions().txStart(conc, isolation, 0, 0)) { fut = rollbackAsync(tx, 200); if (write) @@ -549,7 +558,7 @@ public void testRollbackDelayNearLockRequest() throws Exception { } }, "tx-rollback-thread"); - try(final Transaction tx = client.transactions().txStart()) { + try (final Transaction tx = client.transactions().txStart()) { client.cache(CACHE_NAME).put(0, 0); fail(); @@ -608,7 +617,7 @@ public void testRollbackDelayFinishRequest() throws Exception { } }, "tx-rollback-thread"); - try(final Transaction tx = client.transactions().txStart()) { + try (final Transaction tx = client.transactions().txStart()) { txRef.set(tx); client.cache(CACHE_NAME).put(0, 0); @@ -761,8 +770,8 @@ public void testMixedAsyncRollbackTypes() throws Exception { Transaction tx; // Rollback all transaction - while((tx = nodeQ.poll()) != null) { - rolledBack.add(1); + while ((tx = nodeQ.poll()) != null) { + rolledBack.increment(); doSleep(r.nextInt(50)); // Add random sleep to increase completed txs count. @@ -795,8 +804,8 @@ public void testMixedAsyncRollbackTypes() throws Exception { for (BlockingQueue queue : perNodeTxs.values()) { Transaction tx; - while((tx = queue.poll()) != null) { - rolledBack.add(1); + while ((tx = queue.poll()) != null) { + rolledBack.increment(); rollbackClo.apply(tx); } @@ -942,12 +951,12 @@ public void testRollbackOnTopologyLockPessimistic() throws Exception { @Override public boolean apply(Event evt) { runAsync(new Runnable() { @Override public void run() { - try(Transaction tx = crd.transactions().withLabel("testLbl").txStart()) { + try (Transaction tx = crd.transactions().withLabel("testLbl").txStart()) { // Wait for node start. waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { return crd.cluster().topologyVersion() != GRID_CNT + - /** client node */ 1 + /** stop server node */ 1 + /** start server node */ 1; + /** client node */1 + /** stop server node */1 + /** start server node */1; } }, 10_000); @@ -1023,7 +1032,6 @@ public void testRollbackOnTopologyLockPessimistic() throws Exception { * @param keyLocked Latch for notifying until key is locked. * @param waitCommit Latch for waiting until commit is allowed. * @param timeout Timeout. - * * @return tx completion future. */ private IgniteInternalFuture lockInTx(final Ignite node, final CountDownLatch keyLocked, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnIncorrectParamsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnIncorrectParamsTest.java index 8aafa8b091ba9..75b76257cc8fb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnIncorrectParamsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnIncorrectParamsTest.java @@ -28,6 +28,7 @@ import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -39,6 +40,14 @@ * Tests transaction rollback on incorrect tx params. */ public class TxRollbackOnIncorrectParamsTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10415"); + + super.beforeTestsStarted(); + } + /** * */ @@ -61,14 +70,14 @@ public void testTimeoutSetLocalGuarantee() throws Exception { IgniteCache cache = ignite.getOrCreateCache(defaultCacheConfiguration()); try (Transaction tx = ignite.transactions().txStart( - TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ, 200, 2)) { + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 200, 2)) { cache.put(1, 1); tx.commit(); } try (Transaction tx = ignite.transactions().txStart( - TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ, 100, 2)) { + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 100, 2)) { cache.put(1, 2); tx.commit(); @@ -216,14 +225,14 @@ public void testTimeoutSetRemoteGuarantee() throws Exception { EVT_TX_STARTED); try (Transaction tx = ignite.transactions().txStart( - TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ, 100, 2)) { + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 100, 2)) { cacheLocal.put(1, 1); tx.commit(); } try (Transaction tx = remote.transactions().txStart( - TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ, 100, 2)) { + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 100, 2)) { cacheRemote.put(1, 2); tx.commit(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java index ae75caa1bc74c..8edd0c85e8880 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Random; -import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; @@ -37,8 +36,6 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; -import org.apache.ignite.events.Event; -import org.apache.ignite.events.EventType; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; @@ -54,20 +51,12 @@ import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.visor.VisorTaskArgument; -import org.apache.ignite.internal.visor.tx.VisorTxInfo; -import org.apache.ignite.internal.visor.tx.VisorTxOperation; -import org.apache.ignite.internal.visor.tx.VisorTxTask; -import org.apache.ignite.internal.visor.tx.VisorTxTaskArg; -import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -79,7 +68,6 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.testframework.GridTestUtils.runAsync; -import static org.apache.ignite.testframework.GridTestUtils.runAsync; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; @@ -105,6 +93,14 @@ public class TxRollbackOnTimeoutTest extends GridCommonAbstractTest { /** */ private static final int GRID_CNT = 3; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7388"); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxStateChangeEventTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxStateChangeEventTest.java index 01c87aed7ba5b..446609fdd7907 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxStateChangeEventTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxStateChangeEventTest.java @@ -21,10 +21,14 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteEvents; +import org.apache.ignite.IgniteTransactions; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.events.Event; import org.apache.ignite.events.TransactionStateChangedEvent; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -67,20 +71,20 @@ public class TxStateChangeEventTest extends GridCommonAbstractTest { * */ public void testLocal() throws Exception { - test(true); + check(true); } /** * */ public void testRemote() throws Exception { - test(false); + check(false); } /** * */ - private void test(boolean loc) throws Exception { + private void check(boolean loc) throws Exception { Ignite ignite = startGrids(5); final IgniteEvents evts = loc ? ignite.events() : grid(3).events(); @@ -104,27 +108,51 @@ private void test(boolean loc) throws Exception { }, EVTS_TX); - IgniteCache cache = ignite.getOrCreateCache(defaultCacheConfiguration().setBackups(2)); + IgniteTransactions txs = ignite.transactions(); - // create & commit - try (Transaction tx = ignite.transactions().withLabel(lb).txStart( - TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE, timeout, 3)) { - cache.put(1, 1); + IgniteCache cache = ignite.getOrCreateCache(getCacheConfig()); - tx.commit(); + checkCommit(txs, cache); + + if (!MvccFeatureChecker.forcedMvcc()) + checkSuspendResume(txs, cache); + + checkRollback(txs, cache); + } + + /** */ + @SuppressWarnings("unchecked") + private CacheConfiguration getCacheConfig() { + return defaultCacheConfiguration().setBackups(2); + } + + /** + * @param txs Transaction manager. + * @param cache Ignite cache. + */ + private void checkRollback(IgniteTransactions txs, IgniteCache cache) { + // create & rollback (pessimistic) + try (Transaction tx = txs.withLabel(lb).txStart( + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, timeout, 3)) { + cache.put(4, 5); } assertTrue( creation.get() && - commit.get() && - !rollback.get() && + !commit.get() && + rollback.get() && !suspend.get() && !resume.get()); + } - clear(); - + /** + * @param txs Transaction manager. + * @param cache Ignite cache. + */ + private void checkSuspendResume(IgniteTransactions txs, + IgniteCache cache) throws IgniteInterruptedCheckedException { // create & suspend & resume & commit - try (Transaction tx = ignite.transactions().withLabel(lb).txStart( + try (Transaction tx = txs.withLabel(lb).txStart( TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE, timeout, 3)) { cache.put(2, 7); @@ -145,19 +173,29 @@ private void test(boolean loc) throws Exception { resume.get()); clear(); + } - // create & rollback (pessimistic) - try (Transaction tx = ignite.transactions().withLabel(lb).txStart( - TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE, timeout, 3)) { - cache.put(4, 5); + /** + * @param txs Transaction manager. + * @param cache Ignite cache. + */ + private void checkCommit(IgniteTransactions txs, IgniteCache cache) { + // create & commit + try (Transaction tx = txs.withLabel(lb).txStart( + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, timeout, 3)) { + cache.put(1, 1); + + tx.commit(); } assertTrue( creation.get() && - !commit.get() && - rollback.get() && + commit.get() && + !rollback.get() && !suspend.get() && !resume.get()); + + clear(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index 2ea77b44aee45..07947a94b10ee 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -42,6 +42,7 @@ public enum Feature { CACHE_EVENTS, EVICTION, EXPIRATION, + METRICS INTERCEPTOR } @@ -134,6 +135,9 @@ private static void validateFeature(Feature feature) { case EXPIRATION: fail("https://issues.apache.org/jira/browse/IGNITE-7311"); + case METRICS: + fail("https://issues.apache.org/jira/browse/IGNITE-9224"); + case INTERCEPTOR: fail("https://issues.apache.org/jira/browse/IGNITE-9323"); } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java new file mode 100644 index 0000000000000..688c57252b059 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite6.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import java.util.Set; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.PartitionedMvccTxPessimisticCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.PartitionsExchangeCoordinatorFailoverTest; +import org.apache.ignite.internal.processors.cache.ReplicatedAtomicCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.ReplicatedMvccTxPessimisticCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalOptimisticCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest; +import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheParallelStartTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCache150ClientsTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeMultiServerTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest; +import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPartitionExchangeTest; +import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticPrepareOnUnstableTopologyTest; +import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutOnePhaseCommitTest; +import org.apache.ignite.internal.processors.cache.transactions.TxStateChangeEventTest; + +/** + * Test suite. + */ +public class IgniteCacheMvccTestSuite6 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + Set ignoredTests = new HashSet<>(); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(TxStateChangeEventTest.class); + + // Atomic cache tests. + ignoredTests.add(ReplicatedAtomicCacheGetsDistributionTest.class); + ignoredTests.add(PartitionedAtomicCacheGetsDistributionTest.class); + + // Irrelevant Tx tests. + ignoredTests.add(IgniteOptimisticTxSuspendResumeTest.class); + ignoredTests.add(IgniteOptimisticTxSuspendResumeMultiServerTest.class); + ignoredTests.add(TxOptimisticPrepareOnUnstableTopologyTest.class); + ignoredTests.add(ReplicatedTransactionalOptimisticCacheGetsDistributionTest.class); + ignoredTests.add(PartitionedTransactionalOptimisticCacheGetsDistributionTest.class); + ignoredTests.add(TxOptimisticOnPartitionExchangeTest.class); + + ignoredTests.add(TxRollbackOnTimeoutOnePhaseCommitTest.class); + + // Other non-tx tests. + ignoredTests.add(CacheExchangeMergeTest.class); + ignoredTests.add(IgniteExchangeLatchManagerCoordinatorFailTest.class); + ignoredTests.add(PartitionsExchangeCoordinatorFailoverTest.class); + ignoredTests.add(CacheParallelStartTest.class); + ignoredTests.add(IgniteCache150ClientsTest.class); + + // Skip tests that has Mvcc clones. + ignoredTests.add(PartitionedTransactionalPessimisticCacheGetsDistributionTest.class); // See PartitionedMvccTxPessimisticCacheGetsDistributionTest. + ignoredTests.add(ReplicatedTransactionalPessimisticCacheGetsDistributionTest.class); //See ReplicatedMvccTxPessimisticCacheGetsDistributionTest + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 6"); + + suite.addTest(IgniteCacheTestSuite6.suite(ignoredTests)); + + // Add mvcc versions for skipped tests. + suite.addTestSuite(PartitionedMvccTxPessimisticCacheGetsDistributionTest.class); + suite.addTestSuite(ReplicatedMvccTxPessimisticCacheGetsDistributionTest.class); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java index 7bb476f0b70f7..d4f9729b1438b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.CacheNoAffinityExchangeTest; import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest; @@ -50,6 +51,7 @@ import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest; import org.apache.ignite.internal.processors.cache.transactions.TxStateChangeEventTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -57,64 +59,71 @@ public class IgniteCacheTestSuite6 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 6"); - suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class); - suite.addTestSuite(IgniteOptimisticTxSuspendResumeTest.class); - suite.addTestSuite(IgniteOptimisticTxSuspendResumeMultiServerTest.class); - suite.addTestSuite(IgnitePessimisticTxSuspendResumeTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionEvictionDuringReadThroughSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteOptimisticTxSuspendResumeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteOptimisticTxSuspendResumeMultiServerTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgnitePessimisticTxSuspendResumeTest.class, ignoredTests); - suite.addTestSuite(CacheExchangeMergeTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheExchangeMergeTest.class, ignoredTests); - suite.addTestSuite(TxRollbackOnTimeoutTest.class); - suite.addTestSuite(TxRollbackOnTimeoutNoDeadlockDetectionTest.class); - suite.addTestSuite(TxRollbackOnTimeoutNearCacheTest.class); - suite.addTestSuite(IgniteCacheThreadLocalTxTest.class); - suite.addTestSuite(TxRollbackAsyncTest.class); - suite.addTestSuite(TxRollbackAsyncNearCacheTest.class); - suite.addTestSuite(TxRollbackOnTopologyChangeTest.class); - suite.addTestSuite(TxRollbackOnTimeoutOnePhaseCommitTest.class); + GridTestUtils.addTestIfNeeded(suite, TxRollbackOnTimeoutTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackOnTimeoutNoDeadlockDetectionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackOnTimeoutNearCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheThreadLocalTxTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackAsyncTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackAsyncNearCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackOnTopologyChangeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackOnTimeoutOnePhaseCommitTest.class, ignoredTests); - suite.addTestSuite(TxOptimisticPrepareOnUnstableTopologyTest.class); + GridTestUtils.addTestIfNeeded(suite, TxOptimisticPrepareOnUnstableTopologyTest.class, ignoredTests); - suite.addTestSuite(TxLabelTest.class); - suite.addTestSuite(TxRollbackOnIncorrectParamsTest.class); - suite.addTestSuite(TxStateChangeEventTest.class); + GridTestUtils.addTestIfNeeded(suite, TxLabelTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackOnIncorrectParamsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxStateChangeEventTest.class, ignoredTests); - suite.addTestSuite(TxMultiCacheAsyncOpsTest.class); + GridTestUtils.addTestIfNeeded(suite, TxMultiCacheAsyncOpsTest.class, ignoredTests); - suite.addTestSuite(TxOnCachesStartTest.class); + GridTestUtils.addTestIfNeeded(suite, TxOnCachesStartTest.class, ignoredTests); - suite.addTestSuite(IgniteCache150ClientsTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteCache150ClientsTest.class, ignoredTests); // TODO enable this test after IGNITE-6753, now it takes too long -// suite.addTestSuite(IgniteOutOfMemoryPropagationTest.class); +// GridTestUtils.addTestIfNeeded(suite, IgniteOutOfMemoryPropagationTest.class, ignoredTests); - suite.addTestSuite(ReplicatedAtomicCacheGetsDistributionTest.class); - suite.addTestSuite(ReplicatedTransactionalOptimisticCacheGetsDistributionTest.class); - suite.addTestSuite(ReplicatedTransactionalPessimisticCacheGetsDistributionTest.class); + GridTestUtils.addTestIfNeeded(suite, ReplicatedAtomicCacheGetsDistributionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, ReplicatedTransactionalOptimisticCacheGetsDistributionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, ReplicatedTransactionalPessimisticCacheGetsDistributionTest.class, ignoredTests); - suite.addTestSuite(PartitionedAtomicCacheGetsDistributionTest.class); - suite.addTestSuite(PartitionedTransactionalOptimisticCacheGetsDistributionTest.class); - suite.addTestSuite(PartitionedTransactionalPessimisticCacheGetsDistributionTest.class); + GridTestUtils.addTestIfNeeded(suite, PartitionedAtomicCacheGetsDistributionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PartitionedTransactionalOptimisticCacheGetsDistributionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PartitionedTransactionalPessimisticCacheGetsDistributionTest.class, ignoredTests); - suite.addTestSuite(TxOptimisticOnPartitionExchangeTest.class); + GridTestUtils.addTestIfNeeded(suite, TxOptimisticOnPartitionExchangeTest.class, ignoredTests); - suite.addTestSuite(IgniteExchangeLatchManagerCoordinatorFailTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteExchangeLatchManagerCoordinatorFailTest.class, ignoredTests); - suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class); - suite.addTestSuite(CacheTryLockMultithreadedTest.class); + GridTestUtils.addTestIfNeeded(suite, PartitionsExchangeCoordinatorFailoverTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheTryLockMultithreadedTest.class, ignoredTests); - suite.addTestSuite(CacheParallelStartTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheParallelStartTest.class, ignoredTests); - suite.addTestSuite(CacheNoAffinityExchangeTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheNoAffinityExchangeTest.class, ignoredTests); - //suite.addTestSuite(CacheClientsConcurrentStartTest.class); - //suite.addTestSuite(GridCacheRebalancingOrderingTest.class); - //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class); + //GridTestUtils.addTestIfNeeded(suite, CacheClientsConcurrentStartTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingOrderingTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite, IgniteCacheClientMultiNodeUpdateTopologyLockTest.class, ignoredTests); return suite; } From 1b390af7fd6ba8585693a96a1f4d3e9495dcb1ea Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Fri, 30 Nov 2018 16:10:56 +0300 Subject: [PATCH 194/403] IGNITE-10352 Fix cache get request can be mapped to node while partition in MOVING state - Fixes #5519. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit 18aaee039908821d9d962a7bf3659578164e375d) Signed-off-by: Dmitriy Govorukhin --- .../processors/cache/GridCacheContext.java | 31 +- .../dht/CacheDistributedGetFutureAdapter.java | 436 ++++++++++++- .../distributed/dht/GridDhtCacheAdapter.java | 22 +- .../dht/GridDhtGetSingleFuture.java | 2 +- .../dht/GridPartitionedGetFuture.java | 574 ++++++------------ .../dht/GridPartitionedSingleGetFuture.java | 352 +++++++---- .../distributed/near/GridNearGetFuture.java | 456 +++----------- .../CacheGetReadFromBackupFailoverTest.java | 257 ++++++++ .../testsuites/IgniteCacheTestSuite2.java | 2 + 9 files changed, 1223 insertions(+), 909 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheGetReadFromBackupFailoverTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 664dc7eafee45..31da6b900171a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -30,6 +30,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; @@ -2247,24 +2248,32 @@ else if (type == EVT_CACHE_REBALANCE_STOPPED) { */ @Nullable public ClusterNode selectAffinityNodeBalanced( List affNodes, + Set invalidNodes, int partitionId, boolean canRemap ) { if (!readLoadBalancingEnabled) { if (!canRemap) { + // Find next available node if we can not wait next topology version. for (ClusterNode node : affNodes) { - if (ctx.discovery().alive(node)) + if (ctx.discovery().alive(node) && !invalidNodes.contains(node)) return node; } return null; } - else - return affNodes.get(0); + else { + ClusterNode first = affNodes.get(0); + + return !invalidNodes.contains(first) ? first : null; + } } - if (!readFromBackup) - return affNodes.get(0); + if (!readFromBackup){ + ClusterNode first = affNodes.get(0); + + return !invalidNodes.contains(first) ? first : null; + } assert locMacs != null; @@ -2273,7 +2282,7 @@ else if (type == EVT_CACHE_REBALANCE_STOPPED) { ClusterNode n0 = null; for (ClusterNode node : affNodes) { - if ((canRemap || discovery().alive(node) && isOwner(node, partitionId))) { + if ((canRemap || discovery().alive(node)) && !invalidNodes.contains(node)) { if (locMacs.equals(node.attribute(ATTR_MACS))) return node; @@ -2287,16 +2296,6 @@ else if (type == EVT_CACHE_REBALANCE_STOPPED) { return n0; } - /** - * Check that node is owner for partition. - * @param node Cluster node. - * @param partitionId Partition ID. - * @return {@code} - */ - private boolean isOwner(ClusterNode node, int partitionId) { - return topology().partitionState(node.id(), partitionId) == OWNING; - } - /** * Prepare affinity field for builder (if possible). * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java index 62b5fe9879c58..d0228159a8721 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java @@ -18,18 +18,40 @@ package org.apache.ignite.internal.processors.cache.distributed.dht; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo; import org.apache.ignite.internal.processors.cache.GridCacheFuture; import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.C1; +import org.apache.ignite.internal.util.typedef.CIX1; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.P1; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; @@ -40,8 +62,14 @@ /** * */ -public abstract class CacheDistributedGetFutureAdapter extends GridCacheCompoundIdentityFuture> - implements GridCacheFuture>, CacheGetFuture { +public abstract class CacheDistributedGetFutureAdapter + extends GridCacheCompoundIdentityFuture> implements CacheGetFuture { + /** Logger reference. */ + protected static final AtomicReference logRef = new AtomicReference<>(); + + /** Logger. */ + protected static IgniteLogger log; + /** Default max remap count value. */ public static final int DFLT_MAX_REMAP_CNT = 3; @@ -101,6 +129,10 @@ public abstract class CacheDistributedGetFutureAdapter extends GridCacheCo /** */ protected final boolean recovery; + /** */ + protected Map>> invalidNodes = Collections.emptyMap(); + + /** * @param cctx Context. * @param keys Keys. @@ -149,6 +181,29 @@ protected CacheDistributedGetFutureAdapter( futId = IgniteUuid.randomUuid(); } + /** + * @param aclass Class. + */ + protected void initLogger(Class aclass){ + if (log == null) + log = U.logger(cctx.kernalContext(), logRef, aclass); + } + + /** {@inheritDoc} */ + @Override public boolean trackable() { + return trackable; + } + + /** {@inheritDoc} */ + @Override public void markNotTrackable() { + // Should not flip trackable flag from true to false since get future can be remapped. + } + + /** {@inheritDoc} */ + @Override public IgniteUuid futureId() { + return futId; + } + /** * @param part Partition. * @return {@code True} if partition is in owned state. @@ -158,11 +213,384 @@ protected final boolean partitionOwned(int part) { } /** + * @param fut Future. + */ + protected void registrateFutureInMvccManager(GridCacheFuture fut) { + if (!trackable) { + trackable = true; + + cctx.mvcc().addFuture(fut, futId); + } + } + + /** + * @param node Cluster node. + * @param part Invalid partition. + * @param topVer Topology version. + */ + protected synchronized void addNodeAsInvalid(ClusterNode node, int part, AffinityTopologyVersion topVer) { + if (invalidNodes == Collections.>>emptyMap()) { + invalidNodes = new HashMap<>(); + } + + Map> invalidNodeMap = invalidNodes.get(topVer); + + if (invalidNodeMap == null) + invalidNodes.put(topVer, invalidNodeMap = new HashMap<>()); + + Set invalidNodeSet = invalidNodeMap.get(part); + + if (invalidNodeSet == null) + invalidNodeMap.put(part, invalidNodeSet = new HashSet<>()); + + invalidNodeSet.add(node); + } + + /** + * @param part Partition. + * @param topVer Topology version. + * @return Set of invalid cluster nodes. + */ + protected synchronized Set getInvalidNodes(int part, AffinityTopologyVersion topVer) { + Set invalidNodeSet = Collections.emptySet(); + + Map> invalidNodesMap = invalidNodes.get(topVer); + + if (invalidNodesMap != null) { + Set nodes = invalidNodesMap.get(part); + + if (nodes != null) + invalidNodeSet = nodes; + } + + return invalidNodeSet; + } + + /** + * + * @param key Key. + * @param node Mapped node. + * @param missedNodesToKeysMapping Full node mapping. + */ + protected boolean checkRetryPermits( + KeyCacheObject key, + ClusterNode node, + Map> missedNodesToKeysMapping + ) { + LinkedHashMap keys = missedNodesToKeysMapping.get(node); + + if (keys != null && keys.containsKey(key)) { + if (REMAP_CNT_UPD.incrementAndGet(this) > MAX_REMAP_CNT) { + onDone(new ClusterTopologyCheckedException("Failed to remap key to a new node after " + + MAX_REMAP_CNT + " attempts (key got remapped to the same node) [key=" + key + ", node=" + + U.toShortString(node) + ", mappings=" + missedNodesToKeysMapping + ']')); + + return false; + } + } + + return true; + } + + /** {@inheritDoc} */ + @Override public boolean onNodeLeft(UUID nodeId) { + boolean found = false; + + for (IgniteInternalFuture> fut : futures()) + if (isMini(fut)) { + AbstractMiniFuture f = (AbstractMiniFuture)fut; + + if (f.node().id().equals(nodeId)) { + found = true; + + f.onNodeLeft(new ClusterTopologyCheckedException("Remote node left grid (will retry): " + nodeId)); + } + } + + return found; + } + + /** {@inheritDoc} */ + @Override public void onResult(UUID nodeId, GridNearGetResponse res) { + for (IgniteInternalFuture> fut : futures()) + if (isMini(fut)) { + AbstractMiniFuture f = (AbstractMiniFuture)fut; + + if (f.futureId().equals(res.miniId())) { + assert f.node().id().equals(nodeId); + + f.onResult(res); + } + } + } + + /** + * @param part Partition. * @param topVer Topology version. * @return Exception. */ - protected final ClusterTopologyServerNotFoundException serverNotFoundError(AffinityTopologyVersion topVer) { + protected final ClusterTopologyServerNotFoundException serverNotFoundError(int part, AffinityTopologyVersion topVer) { return new ClusterTopologyServerNotFoundException("Failed to map keys for cache " + - "(all partition nodes left the grid) [topVer=" + topVer + ", cache=" + cctx.name() + ']'); + "(all partition nodes left the grid) [topVer=" + topVer + + ", part" + part + ", cache=" + cctx.name() + ", localNodeId=" + cctx.localNodeId() + ']'); + } + + /** + * @param f Future. + * @return {@code True} if mini-future. + */ + protected abstract boolean isMini(IgniteInternalFuture f); + + /** + * @param keys Collection of mapping keys. + * @param mapped Previous mapping. + * @param topVer Topology version. + */ + protected abstract void map( + Collection keys, + Map> mapped, + AffinityTopologyVersion topVer + ); + + /** {@inheritDoc} */ + @Override public String toString() { + Collection futuresStrings = F.viewReadOnly(futures(), new C1, String>() { + @SuppressWarnings("unchecked") + @Override public String apply(IgniteInternalFuture f) { + if (isMini(f)) { + AbstractMiniFuture mini = (AbstractMiniFuture)f; + + return "miniFuture([futId=" + mini.futureId() + ", node=" + mini.node().id() + + ", loc=" + mini.node().isLocal() + + ", done=" + f.isDone() + "])"; + } + else + return f.getClass().getSimpleName() + " [loc=true, done=" + f.isDone() + "]"; + } + }); + + return S.toString(CacheDistributedGetFutureAdapter.class, this, + "innerFuts", futuresStrings, + "super", super.toString()); + } + + /** + * Mini-future for get operations. Mini-futures are only waiting on a single + * node as opposed to multiple nodes. + */ + protected abstract class AbstractMiniFuture extends GridFutureAdapter> { + /** Mini-future id. */ + private final IgniteUuid futId = IgniteUuid.randomUuid(); + + /** Mapped node. */ + protected final ClusterNode node; + + /** Mapped keys. */ + @GridToStringInclude + protected final LinkedHashMap keys; + + /** Topology version on which this future was mapped. */ + protected final AffinityTopologyVersion topVer; + + /** Post processing closure. */ + private final IgniteInClosure> postProcessingClos; + + /** {@code True} if remapped after node left. */ + private boolean remapped; + + /** + * @param node Node. + * @param keys Keys. + * @param topVer Topology version. + */ + protected AbstractMiniFuture( + ClusterNode node, + LinkedHashMap keys, + AffinityTopologyVersion topVer + ) { + this.node = node; + this.keys = keys; + this.topVer = topVer; + this.postProcessingClos = CU.createBackupPostProcessingClosure( + topVer, log, cctx, null, expiryPlc, readThrough, skipVals); + } + + /** + * @return Future ID. + */ + public IgniteUuid futureId() { + return futId; + } + + /** + * @return Node ID. + */ + public ClusterNode node() { + return node; + } + + /** + * @return Keys. + */ + public Collection keys() { + return keys.keySet(); + } + + /** + * Factory methond for generate request associated with this miniFuture. + * + * @param rootFutId Root future id. + * @return Near get request. + */ + public GridNearGetRequest createGetRequest(IgniteUuid rootFutId) { + return createGetRequest0(rootFutId, futureId()); + } + + /** + * @param rootFutId Root future id. + * @param futId Mini future id. + * @return Near get request. + */ + protected abstract GridNearGetRequest createGetRequest0(IgniteUuid rootFutId, IgniteUuid futId); + + /** + * @param entries Collection of entries. + * @return Map with key value results. + */ + protected abstract Map createResultMap(Collection entries); + + /** + * @param e Error. + */ + public void onResult(Throwable e) { + if (log.isDebugEnabled()) + log.debug("Failed to get future result [fut=" + this + ", err=" + e + ']'); + + // Fail. + onDone(e); + } + + /** + * @param e Failure exception. + */ + public synchronized void onNodeLeft(ClusterTopologyCheckedException e) { + if (remapped) + return; + + remapped = true; + + if (log.isDebugEnabled()) + log.debug("Remote node left grid while sending or waiting for reply (will retry): " + this); + + // Try getting from existing nodes. + if (!canRemap) { + map(keys.keySet(), F.t(node, keys), topVer); + + onDone(Collections.emptyMap()); + } + else { + long maxTopVer = Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion()); + + AffinityTopologyVersion awaitTopVer = new AffinityTopologyVersion(maxTopVer); + + cctx.shared().exchange() + .affinityReadyFuture(awaitTopVer) + .listen((f) -> { + try { + // Remap. + map(keys.keySet(), F.t(node, keys), f.get()); + + onDone(Collections.emptyMap()); + } + catch (IgniteCheckedException ex) { + CacheDistributedGetFutureAdapter.this.onDone(ex); + } + } + ); + } + } + + /** + * @param res Result callback. + */ + public void onResult(GridNearGetResponse res) { + // If error happened on remote node, fail the whole future. + if (res.error() != null) { + onDone(res.error()); + + return; + } + + Collection invalidParts = res.invalidPartitions(); + + // Remap invalid partitions. + if (!F.isEmpty(invalidParts)) { + AffinityTopologyVersion rmtTopVer = res.topologyVersion(); + + for (Integer part : invalidParts) + addNodeAsInvalid(node, part, topVer); + + if (log.isDebugEnabled()) + log.debug("Remapping mini get future [invalidParts=" + invalidParts + ", fut=" + this + ']'); + + if (!canRemap) { + map(F.view(keys.keySet(), new P1() { + @Override public boolean apply(KeyCacheObject key) { + return invalidParts.contains(cctx.affinity().partition(key)); + } + }), F.t(node, keys), topVer); + + postProcessResult(res); + + onDone(createResultMap(res.entries())); + + return; + } + + // Remap after remote version will be finished localy. + cctx.shared().exchange().affinityReadyFuture(rmtTopVer) + .listen(new CIX1>() { + @Override public void applyx( + IgniteInternalFuture fut + ) throws IgniteCheckedException { + AffinityTopologyVersion topVer = fut.get(); + + // This will append new futures to compound list. + map(F.view(keys.keySet(), new P1() { + @Override public boolean apply(KeyCacheObject key) { + return invalidParts.contains(cctx.affinity().partition(key)); + } + }), F.t(node, keys), topVer); + + postProcessResult(res); + + onDone(createResultMap(res.entries())); + } + }); + } + else { + try { + postProcessResult(res); + + onDone(createResultMap(res.entries())); + } + catch (Exception e) { + onDone(e); + } + } + } + + /** + * @param res Response. + */ + protected void postProcessResult(final GridNearGetResponse res) { + if (postProcessingClos != null) + postProcessingClos.apply(res.entries()); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(AbstractMiniFuture.class, this); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java index f69f8132410ad..83af08aa833b6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java @@ -959,12 +959,14 @@ else if (req.needVersion()) res0 = info.value(); } - res = new GridNearSingleGetResponse(ctx.cacheId(), + res = new GridNearSingleGetResponse( + ctx.cacheId(), req.futureId(), null, res0, false, - req.addDeploymentInfo()); + req.addDeploymentInfo() + ); if (info != null && req.skipValues()) res.setContainsValue(); @@ -972,15 +974,14 @@ else if (req.needVersion()) else { AffinityTopologyVersion topVer = ctx.shared().exchange().lastTopologyFuture().initialVersion(); - assert topVer.compareTo(req.topologyVersion()) > 0 : "Wrong ready topology version for " + - "invalid partitions response [topVer=" + topVer + ", req=" + req + ']'; - - res = new GridNearSingleGetResponse(ctx.cacheId(), + res = new GridNearSingleGetResponse( + ctx.cacheId(), req.futureId(), topVer, null, true, - req.addDeploymentInfo()); + req.addDeploymentInfo() + ); } } catch (NodeStoppingException ignored) { @@ -1064,8 +1065,11 @@ protected void processNearGetRequest(final UUID nodeId, final GridNearGetRequest res.error(e); } - if (!F.isEmpty(fut.invalidPartitions())) - res.invalidPartitions(fut.invalidPartitions(), ctx.shared().exchange().lastTopologyFuture().initialVersion()); + if (!F.isEmpty(fut.invalidPartitions())){ + AffinityTopologyVersion topVer = ctx.shared().exchange().lastTopologyFuture().initialVersion(); + + res.invalidPartitions(fut.invalidPartitions(), topVer); + } try { ctx.io().send(nodeId, res, ctx.ioPolicy()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java index ea806635bb123..a8f74cf79fbaa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java @@ -207,7 +207,7 @@ public GridCacheVersion version() { * */ private void map() { - // TODO get rid of force keys request https://issues.apache.org/jira/browse/IGNITE-10251 + // TODO Get rid of force keys request https://issues.apache.org/jira/browse/IGNITE-10251. if (cctx.group().preloader().needForceKeys()) { GridDhtFuture fut = cctx.group().preloader().request( cctx, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java index 9e91d66805e5f..7f2781bdeaece 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java @@ -23,10 +23,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; @@ -43,7 +42,6 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse; import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker; import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTrackerImpl; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; @@ -52,17 +50,10 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.GridLeanMap; import org.apache.ignite.internal.util.future.GridFinishedFuture; -import org.apache.ignite.internal.util.future.GridFutureAdapter; -import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.C1; -import org.apache.ignite.internal.util.typedef.CI1; -import org.apache.ignite.internal.util.typedef.CIX1; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.P1; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; @@ -71,14 +62,6 @@ */ public class GridPartitionedGetFuture extends CacheDistributedGetFutureAdapter implements MvccSnapshotResponseListener { - /** */ - private static final long serialVersionUID = 0L; - - /** Logger reference. */ - private static final AtomicReference logRef = new AtomicReference<>(); - - /** Logger. */ - private static IgniteLogger log; /** */ protected final MvccSnapshot mvccSnapshot; @@ -117,7 +100,8 @@ public GridPartitionedGetFuture( boolean keepCacheObjects, @Nullable MvccSnapshot mvccSnapshot ) { - super(cctx, + super( + cctx, keys, readThrough, forcePrimary, @@ -128,13 +112,14 @@ public GridPartitionedGetFuture( skipVals, needVer, keepCacheObjects, - recovery); + recovery + ); + assert mvccSnapshot == null || cctx.mvccEnabled(); this.mvccSnapshot = mvccSnapshot; - if (log == null) - log = U.logger(cctx.kernalContext(), logRef, GridPartitionedGetFuture.class); + initLogger(GridPartitionedGetFuture.class); } /** @@ -162,14 +147,15 @@ public GridPartitionedGetFuture( public void init(AffinityTopologyVersion topVer) { AffinityTopologyVersion lockedTopVer = cctx.shared().lockedTopologyVersion(null); + // Can not remap if we in transaction and locked on some topology. if (lockedTopVer != null) { topVer = lockedTopVer; canRemap = false; } - else { - topVer = topVer.topologyVersion() > 0 ? topVer : - canRemap ? cctx.affinity().affinityTopologyVersion() : cctx.shared().exchange().readyAffinityVersion(); + else{ + // Use affinity topology version if constructor version is not specify. + topVer = topVer.topologyVersion() > 0 ? topVer : cctx.affinity().affinityTopologyVersion(); } if (!cctx.mvccEnabled() || mvccSnapshot != null) @@ -177,84 +163,33 @@ public void init(AffinityTopologyVersion topVer) { else { mvccTracker = new MvccQueryTrackerImpl(cctx, canRemap); - trackable = true; - - cctx.mvcc().addFuture(this, futId); + registrateFutureInMvccManager(this); mvccTracker.requestSnapshot(topVer, this); } } - @Override public void onResponse(MvccSnapshot res) { - AffinityTopologyVersion topVer = mvccTracker.topologyVersion(); - - assert topVer != null; - - initialMap(topVer); - } - - @Override public void onError(IgniteCheckedException e) { - onDone(e); - } - /** * @param topVer Topology version. */ private void initialMap(AffinityTopologyVersion topVer) { - map(keys, Collections.>emptyMap(), topVer); + map(keys, Collections.emptyMap(), topVer); markInitialized(); } /** {@inheritDoc} */ - @Override public boolean trackable() { - return trackable; - } + @Override public void onResponse(MvccSnapshot res) { + AffinityTopologyVersion topVer = mvccTracker.topologyVersion(); - /** {@inheritDoc} */ - @Override public void markNotTrackable() { - // Should not flip trackable flag from true to false since get future can be remapped. - } + assert topVer != null; - /** {@inheritDoc} */ - @Override public IgniteUuid futureId() { - return futId; + initialMap(topVer); } /** {@inheritDoc} */ - @Override public boolean onNodeLeft(UUID nodeId) { - boolean found = false; - - for (IgniteInternalFuture> fut : futures()) - if (isMini(fut)) { - MiniFuture f = (MiniFuture)fut; - - if (f.node().id().equals(nodeId)) { - found = true; - - f.onNodeLeft(new ClusterTopologyCheckedException("Remote node left grid (will retry): " + nodeId)); - } - } - - return found; - } - - /** - * @param nodeId Sender. - * @param res Result. - */ - @Override public void onResult(UUID nodeId, GridNearGetResponse res) { - for (IgniteInternalFuture> fut : futures()) { - if (isMini(fut)) { - MiniFuture f = (MiniFuture)fut; - - if (f.futureId().equals(res.miniId())) { - assert f.node().id().equals(nodeId); - - f.onResult(res); - } - } - } + @Override public void onError(IgniteCheckedException e) { + onDone(e); } /** {@inheritDoc} */ @@ -263,6 +198,8 @@ private void initialMap(AffinityTopologyVersion topVer) { if (trackable) cctx.mvcc().removeFuture(futId); + MvccQueryTracker mvccTracker = this.mvccTracker; + if (mvccTracker != null) mvccTracker.onDone(); @@ -274,11 +211,8 @@ private void initialMap(AffinityTopologyVersion topVer) { return false; } - /** - * @param f Future. - * @return {@code True} if mini-future. - */ - private boolean isMini(IgniteInternalFuture f) { + /** {@inheritDoc} */ + @Override protected boolean isMini(IgniteInternalFuture f) { return f.getClass().equals(MiniFuture.class); } @@ -287,68 +221,60 @@ private boolean isMini(IgniteInternalFuture f) { * @param mapped Mappings to check for duplicates. * @param topVer Topology version on which keys should be mapped. */ - private void map( + @Override protected void map( Collection keys, Map> mapped, - final AffinityTopologyVersion topVer + AffinityTopologyVersion topVer ) { Collection cacheNodes = CU.affinityNodes(cctx, topVer); - if (cacheNodes.isEmpty()) { - onDone(new ClusterTopologyServerNotFoundException("Failed to map keys for cache " + - "(all partition nodes left the grid) [topVer=" + topVer + ", cache=" + cctx.name() + ']')); - - return; - } - - GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture(); - - Throwable err = topFut != null ? topFut.validateCache(cctx, recovery, true, null, keys) : null; - - if (err != null) { - onDone(err); + validate(cacheNodes, topVer); + // Future can be already done with some exception. + if (isDone()) return; - } Map> mappings = U.newHashMap(cacheNodes.size()); - final int keysSize = keys.size(); + int keysSize = keys.size(); + // Map for local (key,value) pairs. Map locVals = U.newHashMap(keysSize); + // True if we have remote nodes after key mapping complete. boolean hasRmtNodes = false; - // Assign keys to primary nodes. + // Assign keys to nodes. for (KeyCacheObject key : keys) - hasRmtNodes |= map(key, mappings, locVals, topVer, mapped); + hasRmtNodes |= map(key, topVer, mappings, mapped, locVals); + // Future can be alredy done with some exception. if (isDone()) return; + // Add local read (key,value) in result. if (!locVals.isEmpty()) add(new GridFinishedFuture<>(locVals)); - if (hasRmtNodes) { - if (!trackable) { - trackable = true; - - cctx.mvcc().addFuture(this, futId); - } - } + // If we have remote nodes in mapping we should registrate future in mvcc manager. + if (hasRmtNodes) + registrateFutureInMvccManager(this); - // Create mini futures. + // Create mini futures after mapping to remote nodes. for (Map.Entry> entry : mappings.entrySet()) { - final ClusterNode n = entry.getKey(); + // Node for request. + ClusterNode n = entry.getKey(); - final LinkedHashMap mappedKeys = entry.getValue(); + // Keys for request. + LinkedHashMap mappedKeys = entry.getValue(); assert !mappedKeys.isEmpty(); // If this is the primary or backup node for the keys. if (n.isLocal()) { - final GridDhtFuture> fut = - cache().getDhtAsync(n.id(), + GridDhtFuture> fut = cache() + .getDhtAsync( + n.id(), -1, mappedKeys, false, @@ -359,67 +285,50 @@ private void map( expiryPlc, skipVals, recovery, - mvccSnapshot()); + mvccSnapshot() + ); - final Collection invalidParts = fut.invalidPartitions(); + Collection invalidParts = fut.invalidPartitions(); if (!F.isEmpty(invalidParts)) { Collection remapKeys = new ArrayList<>(keysSize); for (KeyCacheObject key : keys) { - if (key != null && invalidParts.contains(cctx.affinity().partition(key))) + int part = cctx.affinity().partition(key); + + if (key != null && invalidParts.contains(part)) { + addNodeAsInvalid(n, part, topVer); + remapKeys.add(key); + } } AffinityTopologyVersion updTopVer = cctx.shared().exchange().readyAffinityVersion(); - assert updTopVer.compareTo(topVer) > 0 : "Got invalid partitions for local node but topology version did " + - "not change [topVer=" + topVer + ", updTopVer=" + updTopVer + - ", invalidParts=" + invalidParts + ']'; - // Remap recursively. map(remapKeys, mappings, updTopVer); } // Add new future. - add(fut.chain(new C1>, Map>() { - @Override public Map apply(IgniteInternalFuture> fut) { - try { - return createResultMap(fut.get()); - } - catch (Exception e) { - U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e); + add(fut.chain(f -> { + try { + return createResultMap(f.get()); + } + catch (Exception e) { + U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e); - onDone(e); + onDone(e); - return Collections.emptyMap(); - } + return Collections.emptyMap(); } })); } else { - MiniFuture fut = new MiniFuture(n, mappedKeys, topVer, - CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, readThrough, skipVals)); - - GridCacheMessage req = new GridNearGetRequest( - cctx.cacheId(), - futId, - fut.futureId(), - null, - mappedKeys, - readThrough, - topVer, - subjId, - taskName == null ? 0 : taskName.hashCode(), - expiryPlc != null ? expiryPlc.forCreate() : -1L, - expiryPlc != null ? expiryPlc.forAccess() : -1L, - false, - skipVals, - cctx.deploymentEnabled(), - recovery, - mvccSnapshot()); + MiniFuture miniFut = new MiniFuture(n, mappedKeys, topVer); - add(fut); // Append new future. + GridCacheMessage req = miniFut.createGetRequest(futId); + + add(miniFut); // Append new future. try { cctx.io().send(n, req, cctx.ioPolicy()); @@ -427,84 +336,119 @@ private void map( catch (IgniteCheckedException e) { // Fail the whole thing. if (e instanceof ClusterTopologyCheckedException) - fut.onNodeLeft((ClusterTopologyCheckedException)e); + miniFut.onNodeLeft((ClusterTopologyCheckedException)e); else - fut.onResult(e); + miniFut.onResult(e); } } } } /** - * @param mappings Mappings. + * @param nodesToKeysMapping Mappings. * @param key Key to map. * @param locVals Local values. * @param topVer Topology version. - * @param mapped Previously mapped. + * @param missedNodesToKeysMapping Previously mapped. * @return {@code True} if has remote nodes. */ @SuppressWarnings("ConstantConditions") private boolean map( KeyCacheObject key, - Map> mappings, - Map locVals, AffinityTopologyVersion topVer, - Map> mapped + Map> nodesToKeysMapping, + Map> missedNodesToKeysMapping, + Map locVals ) { int part = cctx.affinity().partition(key); List affNodes = cctx.affinity().nodesByPartition(part, topVer); + // Failed if none affinity node found. if (affNodes.isEmpty()) { - onDone(serverNotFoundError(topVer)); + onDone(serverNotFoundError(part, topVer)); return false; } - // Local get cannot be used with MVCC as local node can contain some visible version which is not latest. - boolean fastLocGet = !cctx.mvccEnabled() && (!forcePrimary || affNodes.get(0).isLocal()) && - cctx.reserveForFastLocalGet(part, topVer); + // Try to read key localy if we can. + if (tryLocalGet(key, part, topVer, affNodes, locVals)) + return false; - if (fastLocGet) { - try { - if (localGet(topVer, key, part, locVals)) - return false; - } - finally { - cctx.releaseForFastLocalGet(part, topVer); - } - } + Set invalidNodeSet = getInvalidNodes(part, topVer); - ClusterNode node = cctx.selectAffinityNodeBalanced(affNodes, part, canRemap); + // Get remote node for request for this key. + ClusterNode node = cctx.selectAffinityNodeBalanced(affNodes, invalidNodeSet, part, canRemap); + // Failed if none remote node found. if (node == null) { - onDone(serverNotFoundError(topVer)); + onDone(serverNotFoundError(part, topVer)); return false; } + // The node still can be local, see details implementation of #tryLocalGet(). boolean remote = !node.isLocal(); - LinkedHashMap keys = mapped.get(node); + // Check retry counter, bound for avoid inifinit remap. + if (!checkRetryPermits(key, node, missedNodesToKeysMapping)) + return false; - if (keys != null && keys.containsKey(key)) { - if (REMAP_CNT_UPD.incrementAndGet(this) > MAX_REMAP_CNT) { - onDone(new ClusterTopologyCheckedException("Failed to remap key to a new node after " + - MAX_REMAP_CNT + " attempts (key got remapped to the same node) [key=" + key + ", node=" + - U.toShortString(node) + ", mappings=" + mapped + ']')); + addNodeMapping(key, node, nodesToKeysMapping); - return false; - } - } + return remote; + } + /** + * + * @param key Key. + * @param node Mapped node. + * @param mappings Full node mapping. + */ + private void addNodeMapping( + KeyCacheObject key, + ClusterNode node, + Map> mappings + ) { LinkedHashMap old = mappings.get(node); if (old == null) mappings.put(node, old = new LinkedHashMap<>(3, 1f)); old.put(key, false); + } - return remote; + /** + * + * @param key Key. + * @param part Partition. + * @param topVer Topology version. + * @param affNodes Affynity nodes. + * @param locVals Map for local (key,value) pairs. + */ + private boolean tryLocalGet( + KeyCacheObject key, + int part, + AffinityTopologyVersion topVer, + List affNodes, + Map locVals + ) { + // Local get cannot be used with MVCC as local node can contain some visible version which is not latest. + boolean fastLocGet = !cctx.mvccEnabled() && + (!forcePrimary || affNodes.get(0).isLocal()) && + cctx.reserveForFastLocalGet(part, topVer); + + if (fastLocGet) { + try { + if (localGet(topVer, key, part, locVals)) + return true; + } + finally { + cctx.releaseForFastLocalGet(part, topVer); + } + } + + return false; } /** @@ -652,6 +596,27 @@ private boolean localGet(AffinityTopologyVersion topVer, KeyCacheObject key, int } } + /** + * + * @param cacheNodes Cache affynity nodes. + * @param topVer Topology version. + */ + private void validate(Collection cacheNodes, AffinityTopologyVersion topVer) { + if (cacheNodes.isEmpty()) { + onDone(new ClusterTopologyServerNotFoundException("Failed to map keys for cache " + + "(all partition nodes left the grid) [topVer=" + topVer + ", cache=" + cctx.name() + ']')); + + return; + } + + GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture(); + + Throwable err = topFut != null ? topFut.validateCache(cctx, recovery, true, null, keys) : null; + + if (err != null) + onDone(err); + } + /** * @return Near cache. */ @@ -692,21 +657,7 @@ private Map createResultMap(Collection infos) { /** {@inheritDoc} */ @Override public String toString() { - Collection futs = F.viewReadOnly(futures(), new C1, String>() { - @SuppressWarnings("unchecked") - @Override public String apply(IgniteInternalFuture f) { - if (isMini(f)) { - return "[node=" + ((MiniFuture)f).node().id() + - ", loc=" + ((MiniFuture)f).node().isLocal() + - ", done=" + f.isDone() + "]"; - } - else - return "[loc=true, done=" + f.isDone() + "]"; - } - }); - return S.toString(GridPartitionedGetFuture.class, this, - "innerFuts", futs, "super", super.toString()); } @@ -714,200 +665,45 @@ private Map createResultMap(Collection infos) { * Mini-future for get operations. Mini-futures are only waiting on a single * node as opposed to multiple nodes. */ - private class MiniFuture extends GridFutureAdapter> { - /** */ - private final IgniteUuid futId = IgniteUuid.randomUuid(); - - /** Node ID. */ - private final ClusterNode node; - - /** Keys. */ - @GridToStringInclude - private final LinkedHashMap keys; - - /** Topology version on which this future was mapped. */ - private final AffinityTopologyVersion topVer; - - /** Post processing closure. */ - private final IgniteInClosure> postProcessingClos; - - /** {@code True} if remapped after node left. */ - private boolean remapped; - + private class MiniFuture extends AbstractMiniFuture { /** * @param node Node. * @param keys Keys. * @param topVer Topology version. - * @param postProcessingClos Post processing closure. */ - MiniFuture(ClusterNode node, LinkedHashMap keys, AffinityTopologyVersion topVer, - @Nullable IgniteInClosure> postProcessingClos) { - this.node = node; - this.keys = keys; - this.topVer = topVer; - this.postProcessingClos = postProcessingClos; + public MiniFuture( + ClusterNode node, + LinkedHashMap keys, + AffinityTopologyVersion topVer + ) { + super(node, keys, topVer); } - /** - * @return Future ID. - */ - IgniteUuid futureId() { - return futId; - } - - /** - * @return Node ID. - */ - public ClusterNode node() { - return node; - } - - /** - * @return Keys. - */ - public Collection keys() { - return keys.keySet(); - } - - /** - * @param e Error. - */ - void onResult(Throwable e) { - if (log.isDebugEnabled()) - log.debug("Failed to get future result [fut=" + this + ", err=" + e + ']'); - - // Fail. - onDone(e); - } - - /** - * @param e Failure exception. - */ - @SuppressWarnings("UnusedParameters") - synchronized void onNodeLeft(ClusterTopologyCheckedException e) { - if (remapped) - return; - - remapped = true; - - if (log.isDebugEnabled()) - log.debug("Remote node left grid while sending or waiting for reply (will retry): " + this); - - // Try getting from existing nodes. - if (!canRemap) { - map(keys.keySet(), F.t(node, keys), topVer); - - onDone(Collections.emptyMap()); - } - else { - AffinityTopologyVersion updTopVer = - new AffinityTopologyVersion(Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion())); - - cctx.shared().exchange().affinityReadyFuture(updTopVer).listen( - new CI1>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - // Remap. - map(keys.keySet(), F.t(node, keys), fut.get()); - - onDone(Collections.emptyMap()); - } - catch (IgniteCheckedException e) { - GridPartitionedGetFuture.this.onDone(e); - } - } - } - ); - } - } - - /** - * @param res Result callback. - */ - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - void onResult(final GridNearGetResponse res) { - final Collection invalidParts = res.invalidPartitions(); - - // If error happened on remote node, fail the whole future. - if (res.error() != null) { - onDone(res.error()); - - return; - } - - // Remap invalid partitions. - if (!F.isEmpty(invalidParts)) { - AffinityTopologyVersion rmtTopVer = res.topologyVersion(); - - assert !rmtTopVer.equals(AffinityTopologyVersion.ZERO); - - if (rmtTopVer.compareTo(topVer) <= 0) { - // Fail the whole get future. - onDone(new IgniteCheckedException("Failed to process invalid partitions response (remote node reported " + - "invalid partitions but remote topology version does not differ from local) " + - "[topVer=" + topVer + ", rmtTopVer=" + rmtTopVer + ", invalidParts=" + invalidParts + - ", nodeId=" + node.id() + ']')); - - return; - } - - if (log.isDebugEnabled()) - log.debug("Remapping mini get future [invalidParts=" + invalidParts + ", fut=" + this + ']'); - - if (!canRemap) { - map(F.view(keys.keySet(), new P1() { - @Override public boolean apply(KeyCacheObject key) { - return invalidParts.contains(cctx.affinity().partition(key)); - } - }), F.t(node, keys), topVer); - - postProcessResult(res); - - onDone(createResultMap(res.entries())); - - return; - } - - // Need to wait for next topology version to remap. - IgniteInternalFuture topFut = cctx.shared().exchange().affinityReadyFuture(rmtTopVer); - - topFut.listen(new CIX1>() { - @SuppressWarnings("unchecked") - @Override public void applyx( - IgniteInternalFuture fut) throws IgniteCheckedException { - AffinityTopologyVersion topVer = fut.get(); - - // This will append new futures to compound list. - map(F.view(keys.keySet(), new P1() { - @Override public boolean apply(KeyCacheObject key) { - return invalidParts.contains(cctx.affinity().partition(key)); - } - }), F.t(node, keys), topVer); - - postProcessResult(res); - - onDone(createResultMap(res.entries())); - } - }); - } - else { - try { - postProcessResult(res); - - onDone(createResultMap(res.entries())); - } - catch (Exception e) { - onDone(e); - } - } + /** {@inheritDoc} */ + @Override protected GridNearGetRequest createGetRequest0(IgniteUuid rootFutId, IgniteUuid futId) { + return new GridNearGetRequest( + cctx.cacheId(), + rootFutId, + futId, + null, + keys, + readThrough, + topVer, + subjId, + taskName == null ? 0 : taskName.hashCode(), + expiryPlc != null ? expiryPlc.forCreate() : -1L, + expiryPlc != null ? expiryPlc.forAccess() : -1L, + false, + skipVals, + cctx.deploymentEnabled(), + recovery, + mvccSnapshot() + ); } - /** - * @param res Response. - */ - private void postProcessResult(final GridNearGetResponse res) { - if (postProcessingClos != null) - postProcessingClos.apply(res.entries()); + /** {@inheritDoc} */ + @Override protected Map createResultMap(Collection entries) { + return GridPartitionedGetFuture.this.createResultMap(entries); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java index 6ae625c4cb042..60b175a8a76cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java @@ -18,8 +18,12 @@ package org.apache.ignite.internal.processors.cache.distributed.dht; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; @@ -51,8 +55,6 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.CI1; -import org.apache.ignite.internal.util.typedef.CIX1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; @@ -61,13 +63,25 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_NEAR_GET_MAX_REMAPS; +import static org.apache.ignite.IgniteSystemProperties.getInteger; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; /** * */ -public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter implements GridCacheFuture, - CacheGetFuture, IgniteDiagnosticAware { +public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter + implements CacheGetFuture, IgniteDiagnosticAware { + /** Default max remap count value. */ + public static final int DFLT_MAX_REMAP_CNT = 3; + + /** Maximum number of attempts to remap key to the same primary node. */ + protected static final int MAX_REMAP_CNT = getInteger(IGNITE_NEAR_GET_MAX_REMAPS, DFLT_MAX_REMAP_CNT); + + /** Remap count updater. */ + protected static final AtomicIntegerFieldUpdater REMAP_CNT_UPD = + AtomicIntegerFieldUpdater.newUpdater(GridPartitionedSingleGetFuture.class, "remapCnt"); + /** Logger reference. */ private static final AtomicReference logRef = new AtomicReference<>(); @@ -132,6 +146,12 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter invalidNodes = Collections.emptySet(); + + /** Remap count. */ + protected volatile int remapCnt; + /** * @param cctx Context. * @param key Key. @@ -197,30 +217,30 @@ public GridPartitionedSingleGetFuture( } /** - * + * Initialize future. */ public void init() { - AffinityTopologyVersion topVer = this.topVer.topologyVersion() > 0 ? this.topVer : - canRemap ? cctx.affinity().affinityTopologyVersion() : cctx.shared().exchange().readyAffinityVersion(); - - GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture(); - - Throwable err = topFut != null ? topFut.validateCache(cctx, recovery, true, key, null) : null; + AffinityTopologyVersion mappingtopVermappingtopVer; - if (err != null) { - onDone(err); - - return; + if (topVer.topologyVersion() > 0) + mappingtopVermappingtopVer = topVer; + else { + mappingtopVermappingtopVer = canRemap ? + cctx.affinity().affinityTopologyVersion() : + cctx.shared().exchange().readyAffinityVersion(); } - map(topVer); + map(mappingtopVermappingtopVer); } /** * @param topVer Topology version. */ @SuppressWarnings("unchecked") - private void map(final AffinityTopologyVersion topVer) { + private void map(AffinityTopologyVersion topVer) { + if (!validate(topVer)) + return; + ClusterNode node = mapKeyToNode(topVer); if (node == null) { @@ -232,45 +252,45 @@ private void map(final AffinityTopologyVersion topVer) { if (isDone()) return; + // Read value if node is localNode. if (node.isLocal()) { - final GridDhtFuture fut = cctx.dht().getDhtSingleAsync(node.id(), - -1, - key, - false, - readThrough, - topVer, - subjId, - taskName == null ? 0 : taskName.hashCode(), - expiryPlc, - skipVals, - recovery, - mvccSnapshot); - - final Collection invalidParts = fut.invalidPartitions(); + GridDhtFuture fut = cctx.dht() + .getDhtSingleAsync( + node.id(), + -1, + key, + false, + readThrough, + topVer, + subjId, + taskName == null ? 0 : taskName.hashCode(), + expiryPlc, + skipVals, + recovery, + mvccSnapshot + ); + + Collection invalidParts = fut.invalidPartitions(); if (!F.isEmpty(invalidParts)) { - AffinityTopologyVersion updTopVer = cctx.shared().exchange().readyAffinityVersion(); + addNodeAsInvalid(node); - assert updTopVer.compareTo(topVer) > 0 : "Got invalid partitions for local node but topology " + - "version did not change [topVer=" + topVer + ", updTopVer=" + updTopVer + - ", invalidParts=" + invalidParts + ']'; + AffinityTopologyVersion updTopVer = cctx.shared().exchange().readyAffinityVersion(); // Remap recursively. map(updTopVer); } else { - fut.listen(new CI1>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - GridCacheEntryInfo info = fut.get(); + fut.listen(f -> { + try { + GridCacheEntryInfo info = f.get(); - setResult(info); - } - catch (Exception e) { - U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e); + setResult(info); + } + catch (Exception e) { + U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e); - onDone(e); - } + onDone(e); } }); } @@ -283,15 +303,11 @@ private void map(final AffinityTopologyVersion topVer) { this.node = node; } - if (!trackable) { - trackable = true; - - cctx.mvcc().addFuture(this, futId); - } + registrateFutureInMvccManager(this); boolean needVer = this.needVer; - final BackupPostProcessingClosure postClos = CU.createBackupPostProcessingClosure(topVer, log, + BackupPostProcessingClosure postClos = CU.createBackupPostProcessingClosure(topVer, log, cctx, key, expiryPlc, readThrough, skipVals); if (postClos != null) { @@ -301,7 +317,8 @@ private void map(final AffinityTopologyVersion topVer) { postProcessingClos = postClos; } - GridCacheMessage req = new GridNearSingleGetRequest(cctx.cacheId(), + GridCacheMessage req = new GridNearSingleGetRequest( + cctx.cacheId(), futId.localId(), key, readThrough, @@ -315,7 +332,8 @@ private void map(final AffinityTopologyVersion topVer) { needVer, cctx.deploymentEnabled(), recovery, - mvccSnapshot); + mvccSnapshot + ); try { cctx.io().send(node, req, cctx.ioPolicy()); @@ -338,35 +356,58 @@ private void map(final AffinityTopologyVersion topVer) { List affNodes = cctx.affinity().nodesByPartition(part, topVer); + // Failed if none affinity node found by assigment. if (affNodes.isEmpty()) { - onDone(serverNotFoundError(topVer)); + onDone(serverNotFoundError(part, topVer)); return null; } + // Try to read key localy if we can. + if (tryLocalGet(key, part, topVer, affNodes)) + return null; + + ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, getInvalidNodes(), part, canRemap); + + // Failed if none balanced node found. + if (affNode == null) { + onDone(serverNotFoundError(part, topVer)); + + return null; + } + + return affNode; + } + + /** + * + * @param key Key. + * @param part Partition. + * @param topVer Topology version. + * @param affNodes Affynity nodes. + */ + private boolean tryLocalGet( + KeyCacheObject key, + int part, + AffinityTopologyVersion topVer, + List affNodes + ) { // Local get cannot be used with MVCC as local node can contain some visible version which is not latest. - boolean fastLocGet = !cctx.mvccEnabled() && (!forcePrimary || affNodes.get(0).isLocal()) && + boolean fastLocGet = !cctx.mvccEnabled() && + (!forcePrimary || affNodes.get(0).isLocal()) && cctx.reserveForFastLocalGet(part, topVer); if (fastLocGet) { try { - if (localGet(topVer, part)) - return null; + if (localGet(topVer, key, part)) + return true; } finally { cctx.releaseForFastLocalGet(part, topVer); } } - ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, part, canRemap); - - if (affNode == null) { - onDone(serverNotFoundError(topVer)); - - return null; - } - - return affNode; + return false; } /** @@ -374,7 +415,7 @@ private void map(final AffinityTopologyVersion topVer) { * @param part Partition. * @return {@code True} if future completed. */ - private boolean localGet(AffinityTopologyVersion topVer, int part) { + private boolean localGet(AffinityTopologyVersion topVer, KeyCacheObject key, int part) { assert cctx.affinityNode() : this; GridDhtCacheAdapter colocated = cctx.dht(); @@ -511,13 +552,29 @@ private boolean localGet(AffinityTopologyVersion topVer, int part) { } } + /** + * @param fut Future. + */ + private void registrateFutureInMvccManager(GridCacheFuture fut) { + if (!trackable) { + trackable = true; + + cctx.mvcc().addFuture(fut, futId); + } + } + /** * @param nodeId Node ID. * @param res Result. */ public void onResult(UUID nodeId, GridNearSingleGetResponse res) { - if (!processResponse(nodeId) || - !checkError(res.error(), res.invalidPartitions(), res.topologyVersion(), nodeId)) + // Brake here if response from unexpected node. + if (!processResponse(nodeId)) + return; + + // Brake here if exception was throws on remote node or + // parition on remote node is invalid. + if (!checkError(nodeId, res.invalidPartitions(), res.topologyVersion(), res.error())) return; Message res0 = res.result(); @@ -552,13 +609,15 @@ else if (readThrough && res0 instanceof CacheVersionedValue) { } } - /** - * @param nodeId Node ID. - * @param res Result. - */ + /** {@inheritDoc} */ @Override public void onResult(UUID nodeId, GridNearGetResponse res) { - if (!processResponse(nodeId) || - !checkError(res.error(), !F.isEmpty(res.invalidPartitions()), res.topologyVersion(), nodeId)) + // Brake here if response from unexpected node. + if (!processResponse(nodeId)) + return; + + // Brake here if exception was throws on remote node or + // parition on remote node is invalid. + if (!checkError(nodeId, !F.isEmpty(res.invalidPartitions()), res.topologyVersion(), res.error())) return; Collection infos = res.entries(); @@ -591,10 +650,12 @@ private boolean processResponse(UUID nodeId) { * @param nodeId Node ID. * @return {@code True} if should process received response. */ - private boolean checkError(@Nullable IgniteCheckedException err, + private boolean checkError( + UUID nodeId, boolean invalidParts, AffinityTopologyVersion rmtTopVer, - UUID nodeId) { + @Nullable IgniteCheckedException err + ) { if (err != null) { onDone(err); @@ -602,34 +663,10 @@ private boolean checkError(@Nullable IgniteCheckedException err, } if (invalidParts) { - assert !rmtTopVer.equals(AffinityTopologyVersion.ZERO); - - if (rmtTopVer.compareTo(topVer) <= 0) { - // Fail the whole get future. - onDone(new IgniteCheckedException("Failed to process invalid partitions response (remote node reported " + - "invalid partitions but remote topology version does not differ from local) " + - "[topVer=" + topVer + ", rmtTopVer=" + rmtTopVer + ", part=" + cctx.affinity().partition(key) + - ", nodeId=" + nodeId + ']')); - - return false; - } + addNodeAsInvalid(cctx.node(nodeId)); if (canRemap) { - IgniteInternalFuture topFut = cctx.shared().exchange().affinityReadyFuture(rmtTopVer); - - topFut.listen(new CIX1>() { - @Override public void applyx(IgniteInternalFuture fut) { - try { - AffinityTopologyVersion topVer = fut.get(); - - remap(topVer); - } - catch (IgniteCheckedException e) { - onDone(e); - } - } - }); - + awaitVersionAndRemap(rmtTopVer); } else map(topVer); @@ -718,12 +755,71 @@ private boolean partitionOwned(int part) { } /** + * @param node Invalid node. + */ + private synchronized void addNodeAsInvalid(ClusterNode node) { + if (invalidNodes == Collections.emptySet()) + invalidNodes = new HashSet<>(); + + invalidNodes.add(node); + } + + /** + * @return Set of invalid cluster nodes. + */ + protected synchronized Set getInvalidNodes() { + return invalidNodes; + } + + /** + * @param topVer Topology version. + */ + private boolean checkRetryPermits(AffinityTopologyVersion topVer) { + if (topVer.equals(this.topVer)) + return true; + + if (REMAP_CNT_UPD.incrementAndGet(this) > MAX_REMAP_CNT) { + ClusterNode node0 = node; + + onDone(new ClusterTopologyCheckedException("Failed to remap key to a new node after " + + MAX_REMAP_CNT + " attempts (key got remapped to the same node) [key=" + key + ", node=" + + (node0 != null ? U.toShortString(node0) : node0) + ", invalidNodes=" + invalidNodes + ']')); + + return false; + } + + return true; + } + + /** + * @param part Partition. * @param topVer Topology version. * @return Exception. */ - private ClusterTopologyServerNotFoundException serverNotFoundError(AffinityTopologyVersion topVer) { + private ClusterTopologyServerNotFoundException serverNotFoundError(int part, AffinityTopologyVersion topVer) { return new ClusterTopologyServerNotFoundException("Failed to map keys for cache " + - "(all partition nodes left the grid) [topVer=" + topVer + ", cache=" + cctx.name() + ']'); + "(all partition nodes left the grid) [topVer=" + topVer + ", part=" + part + ", cache=" + cctx.name() + ']'); + } + + /** + * @param topVer Topology version. + * @return True if validate success, False is not. + */ + private boolean validate(AffinityTopologyVersion topVer) { + if (!checkRetryPermits(topVer)) + return false; + + GridDhtTopologyFuture lastFut = cctx.shared().exchange().lastFinishedFuture(); + + Throwable error = lastFut.validateCache(cctx, recovery, true, key, null); + + if (error != null) { + onDone(error); + + return false; + } + else + return true; } /** {@inheritDoc} */ @@ -737,20 +833,9 @@ private ClusterTopologyServerNotFoundException serverNotFoundError(AffinityTopol return false; if (canRemap) { - AffinityTopologyVersion updTopVer = new AffinityTopologyVersion( - Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion())); - - cctx.shared().exchange().affinityReadyFuture(updTopVer).listen( - new CI1>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - remap(fut.get()); - } - catch (IgniteCheckedException e) { - onDone(e); - } - } - }); + long maxTopVer = Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion()); + + awaitVersionAndRemap(new AffinityTopologyVersion(maxTopVer)); } else remap(topVer); @@ -758,18 +843,33 @@ private ClusterTopologyServerNotFoundException serverNotFoundError(AffinityTopol return true; } + /** + * @param topVer Topology version. + */ + private void awaitVersionAndRemap(AffinityTopologyVersion topVer){ + IgniteInternalFuture awaitTopologyVersionFuture = + cctx.shared().exchange().affinityReadyFuture(topVer); + + awaitTopologyVersionFuture.listen(f -> { + try { + remap(f.get()); + } + catch (IgniteCheckedException e) { + onDone(e); + } + }); + } + /** * @param topVer Topology version. */ private void remap(final AffinityTopologyVersion topVer) { cctx.closures().runLocalSafe(new Runnable() { @Override public void run() { - GridDhtTopologyFuture lastFut = cctx.shared().exchange().lastFinishedFuture(); - - Throwable error = lastFut.validateCache(cctx, recovery, true, key, null); - - if (error != null) - onDone(error); + // If topology changed reset collection of invalid nodes. + synchronized (this) { + invalidNodes = Collections.emptySet(); + } map(topVer); } @@ -823,6 +923,6 @@ private void remap(final AffinityTopologyVersion topVer) { /** {@inheritDoc} */ @Override public String toString() { - return S.toString(GridPartitionedSingleGetFuture.class, this); + return S.toString(GridPartitionedSingleGetFuture.class, this, "super", super.toString()); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java index b954a0b82b9db..308b43e0d658b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java @@ -23,10 +23,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; @@ -38,8 +37,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo; import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException; -import org.apache.ignite.internal.processors.cache.GridCacheMessage; -import org.apache.ignite.internal.processors.cache.GridCacheUtils.BackupPostProcessingClosure; import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.distributed.dht.CacheDistributedGetFutureAdapter; @@ -50,13 +47,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.GridLeanMap; import org.apache.ignite.internal.util.future.GridFinishedFuture; -import org.apache.ignite.internal.util.future.GridFutureAdapter; -import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.C1; -import org.apache.ignite.internal.util.typedef.CI1; -import org.apache.ignite.internal.util.typedef.CIX1; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.P1; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -67,17 +58,8 @@ * */ public final class GridNearGetFuture extends CacheDistributedGetFutureAdapter { - /** */ - private static final long serialVersionUID = 0L; - - /** Logger reference. */ - private static final AtomicReference logRef = new AtomicReference<>(); - - /** Logger. */ - private static IgniteLogger log; - /** Transaction. */ - private IgniteTxLocalEx tx; + private final IgniteTxLocalEx tx; /** */ private GridCacheVersion ver; @@ -112,7 +94,8 @@ public GridNearGetFuture( boolean keepCacheObjects, boolean recovery ) { - super(cctx, + super( + cctx, keys, readThrough, forcePrimary, @@ -123,18 +106,16 @@ public GridNearGetFuture( skipVals, needVer, keepCacheObjects, - recovery); + recovery + ); assert !F.isEmpty(keys); this.tx = tx; - futId = IgniteUuid.randomUuid(); - ver = tx == null ? cctx.versions().next() : tx.xidVersion(); - if (log == null) - log = U.logger(cctx.kernalContext(), logRef, GridNearGetFuture.class); + initLogger(GridNearGetFuture.class); } /** @@ -148,73 +129,21 @@ public void init(@Nullable AffinityTopologyVersion topVer) { if (lockedTopVer != null) { canRemap = false; - map(keys, Collections.>emptyMap(), lockedTopVer); + map(keys, Collections.emptyMap(), lockedTopVer); } else { AffinityTopologyVersion mapTopVer = topVer; if (mapTopVer == null) { - mapTopVer = tx == null ? - (canRemap ? cctx.affinity().affinityTopologyVersion() : cctx.shared().exchange().readyAffinityVersion()) : - tx.topologyVersion(); + mapTopVer = tx == null ? cctx.affinity().affinityTopologyVersion() : tx.topologyVersion(); } - map(keys, Collections.>emptyMap(), mapTopVer); + map(keys, Collections.emptyMap(), mapTopVer); } markInitialized(); } - /** {@inheritDoc} */ - @Override public boolean trackable() { - return trackable; - } - - /** {@inheritDoc} */ - @Override public void markNotTrackable() { - // Should not flip trackable flag from true to false since get future can be remapped. - } - - /** {@inheritDoc} */ - @Override public IgniteUuid futureId() { - return futId; - } - - /** {@inheritDoc} */ - @Override public boolean onNodeLeft(UUID nodeId) { - boolean found = false; - - for (IgniteInternalFuture> fut : futures()) - if (isMini(fut)) { - MiniFuture f = (MiniFuture)fut; - - if (f.node().id().equals(nodeId)) { - found = true; - - f.onNodeLeft(); - } - } - - return found; - } - - /** - * @param nodeId Sender. - * @param res Result. - */ - @Override public void onResult(UUID nodeId, GridNearGetResponse res) { - for (IgniteInternalFuture> fut : futures()) - if (isMini(fut)) { - MiniFuture f = (MiniFuture)fut; - - if (f.futureId().equals(res.miniId())) { - assert f.node().id().equals(nodeId); - - f.onResult(res); - } - } - } - /** {@inheritDoc} */ @Override public boolean onDone(Map res, Throwable err) { if (super.onDone(res, err)) { @@ -230,11 +159,8 @@ public void init(@Nullable AffinityTopologyVersion topVer) { return false; } - /** - * @param f Future. - * @return {@code True} if mini-future. - */ - private boolean isMini(IgniteInternalFuture f) { + /** {@inheritDoc} */ + @Override protected boolean isMini(IgniteInternalFuture f) { return f.getClass().equals(MiniFuture.class); } @@ -243,10 +169,10 @@ private boolean isMini(IgniteInternalFuture f) { * @param mapped Mappings to check for duplicates. * @param topVer Topology version to map on. */ - private void map( + @Override protected void map( Collection keys, Map> mapped, - final AffinityTopologyVersion topVer + AffinityTopologyVersion topVer ) { Collection affNodes = CU.affinityNodes(cctx, topVer); @@ -269,7 +195,7 @@ private void map( try { // Assign keys to primary nodes. for (KeyCacheObject key : keys) - savedEntries = map(key, mappings, topVer, mapped, savedEntries); + savedEntries = map(key, topVer, mappings, mapped, savedEntries); success = true; } @@ -293,23 +219,24 @@ private void map( if (isDone()) return; - final Map saved = savedEntries != null ? savedEntries : - Collections.emptyMap(); + Map saved = + savedEntries != null ? savedEntries : Collections.emptyMap(); - final int keysSize = keys.size(); + int keysSize = keys.size(); // Create mini futures. for (Map.Entry> entry : mappings.entrySet()) { - final ClusterNode n = entry.getKey(); + ClusterNode n = entry.getKey(); - final LinkedHashMap mappedKeys = entry.getValue(); + LinkedHashMap mappedKeys = entry.getValue(); assert !mappedKeys.isEmpty(); // If this is the primary or backup node for the keys. if (n.isLocal()) { - final GridDhtFuture> fut = - dht().getDhtAsync(n.id(), + GridDhtFuture> fut = dht() + .getDhtAsync( + n.id(), -1, mappedKeys, false, @@ -320,73 +247,52 @@ private void map( expiryPlc, skipVals, recovery, - null); // TODO IGNITE-7371 + null + ); // TODO IGNITE-7371 - final Collection invalidParts = fut.invalidPartitions(); + Collection invalidParts = fut.invalidPartitions(); if (!F.isEmpty(invalidParts)) { Collection remapKeys = new ArrayList<>(keysSize); for (KeyCacheObject key : keys) { - if (key != null && invalidParts.contains(cctx.affinity().partition(key))) + int part = cctx.affinity().partition(key); + + if (key != null && invalidParts.contains(part)) { + addNodeAsInvalid(n, part, topVer); + remapKeys.add(key); + } } AffinityTopologyVersion updTopVer = cctx.shared().exchange().readyAffinityVersion(); - assert updTopVer.compareTo(topVer) > 0 : "Got invalid partitions for local node but topology version did " + - "not change [topVer=" + topVer + ", updTopVer=" + updTopVer + - ", invalidParts=" + invalidParts + ']'; - // Remap recursively. map(remapKeys, mappings, updTopVer); } // Add new future. - add(fut.chain(new C1>, Map>() { - @Override public Map apply(IgniteInternalFuture> fut) { - try { - return loadEntries(n.id(), mappedKeys.keySet(), fut.get(), saved, topVer); - } - catch (Exception e) { - U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e); + add(fut.chain(f -> { + try { + return loadEntries(n.id(), mappedKeys.keySet(), f.get(), saved, topVer); + } + catch (Exception e) { + U.error(log, "Failed to get values from dht cache [fut=" + fut + "]", e); - onDone(e); + onDone(e); - return Collections.emptyMap(); - } + return Collections.emptyMap(); } })); } else { - if (!trackable) { - trackable = true; + registrateFutureInMvccManager(this); - cctx.mvcc().addFuture(this, futId); - } + MiniFuture miniFuture = new MiniFuture(n, mappedKeys, saved, topVer); - MiniFuture fut = new MiniFuture(n, mappedKeys, saved, topVer, - CU.createBackupPostProcessingClosure(topVer, log, cctx, null, expiryPlc, readThrough, skipVals)); - - GridCacheMessage req = new GridNearGetRequest( - cctx.cacheId(), - futId, - fut.futureId(), - ver, - mappedKeys, - readThrough, - topVer, - subjId, - taskName == null ? 0 : taskName.hashCode(), - expiryPlc != null ? expiryPlc.forCreate() : -1L, - expiryPlc != null ? expiryPlc.forAccess() : -1L, - true, - skipVals, - cctx.deploymentEnabled(), - recovery, - null); // TODO IGNITE-7371 - - add(fut); // Append new future. + GridNearGetRequest req = miniFuture.createGetRequest(futId); + + add(miniFuture); // Append new future. try { cctx.io().send(n, req, cctx.ioPolicy()); @@ -394,16 +300,14 @@ private void map( catch (IgniteCheckedException e) { // Fail the whole thing. if (e instanceof ClusterTopologyCheckedException) - fut.onNodeLeft(); + miniFuture.onNodeLeft((ClusterTopologyCheckedException)e); else - fut.onResult(e); + miniFuture.onResult(e); } } } } - - /** * @param mappings Mappings. * @param key Key to map. @@ -415,8 +319,8 @@ private void map( @SuppressWarnings("unchecked") private Map map( KeyCacheObject key, - Map> mappings, AffinityTopologyVersion topVer, + Map> mappings, Map> mapped, Map saved ) { @@ -425,7 +329,7 @@ private Map map( List affNodes = cctx.affinity().nodesByPartition(part, topVer); if (affNodes.isEmpty()) { - onDone(serverNotFoundError(topVer)); + onDone(serverNotFoundError(part, topVer)); return null; } @@ -494,10 +398,12 @@ private Map map( } } - ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, part, canRemap); + Set invalidNodesSet = getInvalidNodes(part, topVer); + + ClusterNode affNode = cctx.selectAffinityNodeBalanced(affNodes, invalidNodesSet, part, canRemap); if (affNode == null) { - onDone(serverNotFoundError(topVer)); + onDone(serverNotFoundError(part, topVer)); return saved; } @@ -505,17 +411,8 @@ private Map map( if (cctx.statisticsEnabled() && !skipVals && !affNode.isLocal() && !isNear) cache().metrics0().onRead(false); - LinkedHashMap keys = mapped.get(affNode); - - if (keys != null && keys.containsKey(key)) { - if (REMAP_CNT_UPD.incrementAndGet(this) > MAX_REMAP_CNT) { - onDone(new ClusterTopologyCheckedException("Failed to remap key to a new node after " + - MAX_REMAP_CNT + " attempts (key got remapped to the same node) " + - "[key=" + key + ", node=" + U.toShortString(affNode) + ", mappings=" + mapped + ']')); - - return saved; - } - } + if (!checkRetryPermits(key,affNode,mapped)) + return saved; if (!affNodes.contains(cctx.localNode())) { GridNearCacheEntry nearEntry = entry != null ? entry : near.entryExx(key, topVer); @@ -572,10 +469,12 @@ private Map map( * @param nearRead {@code True} if already tried to read from near cache. * @return {@code True} if there is no need to further search value. */ - private boolean localDhtGet(KeyCacheObject key, + private boolean localDhtGet( + KeyCacheObject key, int part, AffinityTopologyVersion topVer, - boolean nearRead) { + boolean nearRead + ) { GridDhtCacheAdapter dht = cache().dht(); assert dht.context().affinityNode() : this; @@ -795,9 +694,11 @@ private Map loadEntries( * @param saved Saved entries. * @param topVer Topology version. */ - private void releaseEvictions(Collection keys, + private void releaseEvictions( + Collection keys, Map saved, - AffinityTopologyVersion topVer) { + AffinityTopologyVersion topVer + ) { for (KeyCacheObject key : keys) { GridNearCacheEntry entry = saved.get(key); @@ -812,101 +713,58 @@ private void releaseEvictions(Collection keys, /** {@inheritDoc} */ @Override public String toString() { - Collection futs = F.viewReadOnly(futures(), new C1, String>() { - @SuppressWarnings("unchecked") - @Override public String apply(IgniteInternalFuture f) { - if (isMini(f)) { - return "[node=" + ((MiniFuture)f).node().id() + - ", loc=" + ((MiniFuture)f).node().isLocal() + - ", done=" + f.isDone() + "]"; - } - else - return "[loc=true, done=" + f.isDone() + "]"; - } - }); - return S.toString(GridNearGetFuture.class, this, - "innerFuts", futs, "super", super.toString()); } /** - * Mini-future for get operations. Mini-futures are only waiting on a single - * node as opposed to multiple nodes. + * Mini-future for get operations. Mini-futures are only waiting on a single node as opposed to multiple nodes. */ - private class MiniFuture extends GridFutureAdapter> { - /** */ - private final IgniteUuid futId = IgniteUuid.randomUuid(); - - /** Node ID. */ - private ClusterNode node; - - /** Keys. */ - @GridToStringInclude - private LinkedHashMap keys; - + private class MiniFuture extends AbstractMiniFuture { /** Saved entry versions. */ - private Map savedEntries; - - /** Topology version on which this future was mapped. */ - private AffinityTopologyVersion topVer; - - /** Post processing closure. */ - private final BackupPostProcessingClosure postProcessingClos; - - /** {@code True} if remapped after node left. */ - private boolean remapped; + private final Map savedEntries; /** * @param node Node. * @param keys Keys. * @param savedEntries Saved entries. * @param topVer Topology version. - * @param postProcessingClos Post processing closure. */ MiniFuture( ClusterNode node, LinkedHashMap keys, Map savedEntries, - AffinityTopologyVersion topVer, - BackupPostProcessingClosure postProcessingClos) { - this.node = node; - this.keys = keys; + AffinityTopologyVersion topVer + ) { + super(node, keys, topVer); this.savedEntries = savedEntries; - this.topVer = topVer; - this.postProcessingClos = postProcessingClos; } - /** - * @return Future ID. - */ - IgniteUuid futureId() { - return futId; - } - - /** - * @return Node ID. - */ - public ClusterNode node() { - return node; - } - - /** - * @return Keys. - */ - public Collection keys() { - return keys.keySet(); + /** {@inheritDoc} */ + @Override protected GridNearGetRequest createGetRequest0(IgniteUuid rootFutId, IgniteUuid futId) { + return new GridNearGetRequest( + cctx.cacheId(), + rootFutId, + futId, + ver, + keys, + readThrough, + topVer, + subjId, + taskName == null ? 0 : taskName.hashCode(), + expiryPlc != null ? expiryPlc.forCreate() : -1L, + expiryPlc != null ? expiryPlc.forAccess() : -1L, + true, + skipVals, + cctx.deploymentEnabled(), + recovery, + null + ); // TODO IGNITE-7371 } - /** - * @param e Error. - */ - void onResult(Throwable e) { - if (log.isDebugEnabled()) - log.debug("Failed to get future result [fut=" + this + ", err=" + e + ']'); - - // Fail. - onDone(e); + /** {@inheritDoc} */ + @Override protected Map createResultMap(Collection entries) { + return loadEntries(node.id(), keys.keySet(), entries, savedEntries, topVer); } /** {@inheritDoc} */ @@ -920,136 +778,6 @@ void onResult(Throwable e) { return false; } - /** - */ - synchronized void onNodeLeft() { - if (remapped) - return; - - remapped = true; - - if (log.isDebugEnabled()) - log.debug("Remote node left grid while sending or waiting for reply (will retry): " + this); - - // Try getting value from alive nodes. - if (!canRemap) { - // Remap - map(keys.keySet(), F.t(node, keys), topVer); - - onDone(Collections.emptyMap()); - } - else { - AffinityTopologyVersion updTopVer = - new AffinityTopologyVersion(Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion())); - - cctx.shared().exchange().affinityReadyFuture(updTopVer).listen(f -> { - try { - // Remap. - map(keys.keySet(), F.t(node, keys), f.get()); - - onDone(Collections.emptyMap()); - - } - catch (IgniteCheckedException e) { - GridNearGetFuture.this.onDone(e); - } - }); - } - } - - /** - * @param res Result callback. - */ - void onResult(final GridNearGetResponse res) { - final Collection invalidParts = res.invalidPartitions(); - - // If error happened on remote node, fail the whole future. - if (res.error() != null) { - onDone(res.error()); - - return; - } - - // Remap invalid partitions. - if (!F.isEmpty(invalidParts)) { - AffinityTopologyVersion rmtTopVer = res.topologyVersion(); - - assert rmtTopVer.topologyVersion() != 0; - - if (rmtTopVer.compareTo(topVer) <= 0) { - // Fail the whole get future. - onDone(new IgniteCheckedException("Failed to process invalid partitions response (remote node reported " + - "invalid partitions but remote topology version does not differ from local) " + - "[topVer=" + topVer + ", rmtTopVer=" + rmtTopVer + ", invalidParts=" + invalidParts + - ", nodeId=" + node.id() + ']')); - - return; - } - - if (log.isDebugEnabled()) - log.debug("Remapping mini get future [invalidParts=" + invalidParts + ", fut=" + this + ']'); - - if (!canRemap) { - map(F.view(keys.keySet(), new P1() { - @Override public boolean apply(KeyCacheObject key) { - return invalidParts.contains(cctx.affinity().partition(key)); - } - }), F.t(node, keys), topVer); - - postProcessResultAndDone(res); - - return; - } - - // Need to wait for next topology version to remap. - IgniteInternalFuture topFut = cctx.shared().exchange().affinityReadyFuture(rmtTopVer); - - topFut.listen(new CIX1>() { - @Override public void applyx( - IgniteInternalFuture fut) throws IgniteCheckedException { - AffinityTopologyVersion readyTopVer = fut.get(); - - // This will append new futures to compound list. - map(F.view(keys.keySet(), new P1() { - @Override public boolean apply(KeyCacheObject key) { - return invalidParts.contains(cctx.affinity().partition(key)); - } - }), F.t(node, keys), readyTopVer); - - postProcessResultAndDone(res); - } - }); - } - else - postProcessResultAndDone(res); - - } - - /** - * Post processes result and done future. - * - * @param res Response. - */ - private void postProcessResultAndDone(final GridNearGetResponse res){ - try { - postProcessResult(res); - - // It is critical to call onDone after adding futures to compound list. - onDone(loadEntries(node.id(), keys.keySet(), res.entries(), savedEntries, topVer)); - } - catch (Exception ex) { - onDone(ex); - } - } - - /** - * @param res Response. - */ - private void postProcessResult(final GridNearGetResponse res) { - if (postProcessingClos != null) - postProcessingClos.apply(res.entries()); - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(MiniFuture.class, this); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheGetReadFromBackupFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheGetReadFromBackupFailoverTest.java new file mode 100644 index 0000000000000..71cb4ed4b6cba --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheGetReadFromBackupFailoverTest.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.AbstractFailureHandler; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteNodeAttributes; +import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.processors.cache.GridCacheFuture; +import org.apache.ignite.internal.processors.cache.GridCacheMvccManager; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Assert; + +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * Test for getting values on unstable topology with read from backup enabled. + */ +public class CacheGetReadFromBackupFailoverTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** Tx cache name. */ + private static final String TX_CACHE = "txCache"; + /** Atomic cache name. */ + private static final String ATOMIC_CACHE = "atomicCache"; + /** Keys count. */ + private static final int KEYS_CNT = 50000; + /** Stop load flag. */ + private static final AtomicBoolean stop = new AtomicBoolean(); + /** Error. */ + private static final AtomicReference err = new AtomicReference<>(); + + /** + * @return Grid count. + */ + public int gridCount() { + return 3; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setFailureHandler(new AbstractFailureHandler() { + @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) { + err.compareAndSet(null, failureCtx.error()); + stop.set(true); + return false; + } + }); + + cfg.setConsistentId(igniteInstanceName); + + cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER)); + + CacheConfiguration txCcfg = new CacheConfiguration(TX_CACHE) + .setAtomicityMode(TRANSACTIONAL) + .setCacheMode(PARTITIONED) + .setBackups(1) + .setWriteSynchronizationMode(FULL_SYNC) + .setReadFromBackup(true); + + CacheConfiguration atomicCcfg = new CacheConfiguration(ATOMIC_CACHE) + .setAtomicityMode(ATOMIC) + .setCacheMode(PARTITIONED) + .setBackups(1) + .setWriteSynchronizationMode(FULL_SYNC) + .setReadFromBackup(true); + + cfg.setCacheConfiguration(txCcfg, atomicCcfg); + + // Enforce different mac adresses to emulate distributed environment by default. + cfg.setUserAttributes(Collections.singletonMap( + IgniteNodeAttributes.ATTR_MACS_OVERRIDE, UUID.randomUUID().toString())); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + stop.set(false); + + err.set(null); + + startGrids(gridCount()); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * @throws Exception If failed. + */ + public void testFailover() throws Exception { + Ignite ignite = ignite(0); + + ignite.cluster().active(true); + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + try (IgniteDataStreamer stmr = ignite.dataStreamer(TX_CACHE)) { + for (int i = 0; i < KEYS_CNT; i++) + stmr.addData(i, rnd.nextLong()); + } + + try (IgniteDataStreamer stmr = ignite.dataStreamer(ATOMIC_CACHE)) { + for (int i = 0; i < KEYS_CNT; i++) + stmr.addData(i, rnd.nextLong()); + } + + AtomicInteger idx = new AtomicInteger(-1); + + AtomicInteger successGet = new AtomicInteger(); + + IgniteInternalFuture fut = GridTestUtils.runAsync(() -> { + ThreadLocalRandom rnd0 = ThreadLocalRandom.current(); + + while (!stop.get()) { + Ignite ig = null; + + while (ig == null) { + int n = rnd0.nextInt(gridCount()); + + if (idx.get() != n) { + try { + ig = ignite(n); + } + catch (IgniteIllegalStateException e) { + // No-op. + } + } + } + + try { + if (rnd.nextBoolean()) { + ig.cache(TX_CACHE).get(rnd0.nextLong(KEYS_CNT)); + ig.cache(ATOMIC_CACHE).get(rnd0.nextLong(KEYS_CNT)); + } + else { + ig.cache(TX_CACHE).getAll(rnd.longs(16, 0, KEYS_CNT).boxed().collect(Collectors.toSet())); + ig.cache(ATOMIC_CACHE).getAll(rnd.longs(16, 0, KEYS_CNT).boxed().collect(Collectors.toSet())); + } + + successGet.incrementAndGet(); + } + catch (CacheException e) { + if (!X.hasCause(e, NodeStoppingException.class)) + throw e; + } + + } + }, "load-thread"); + + long startTime = System.currentTimeMillis(); + + while (System.currentTimeMillis() - startTime < 30 * 1000L) { + int idx0 = idx.get(); + + if (idx0 >= 0) + startGrid(idx0); + + U.sleep(500); + + int next = rnd.nextInt(gridCount()); + + idx.set(next); + + stopGrid(next); + + U.sleep(500); + } + + stop.set(true); + + while (true){ + try { + fut.get(10_000); + + break; + } + catch (IgniteFutureTimeoutCheckedException e) { + for (Ignite i : G.allGrids()) { + IgniteEx ex = (IgniteEx)i; + + log.info(">>>> " + ex.context().localNodeId()); + + GridCacheMvccManager mvcc = ex.context().cache().context().mvcc(); + + for (GridCacheFuture fut0 : mvcc.activeFutures()) { + log.info("activeFut - " + fut0); + } + + for (GridCacheFuture fut0 : mvcc.atomicFutures()) { + log.info("atomicFut - " + fut0); + } + } + } + } + + Assert.assertTrue(String.valueOf(successGet.get()), successGet.get() > 50); + + Throwable e = err.get(); + + if (e != null) { + log.error("Test failed", e); + + fail("Test failed"); + } + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index 0250d58d386a6..e367aad4e4930 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -66,6 +66,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheClientNodeChangingTopologyTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheClientNodePartitionsExchangeTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheServerNodeConcurrentStart; +import org.apache.ignite.internal.processors.cache.distributed.dht.CacheGetReadFromBackupFailoverTest; import org.apache.ignite.internal.processors.cache.distributed.dht.CachePartitionPartialCountersMapSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedOptimisticTransactionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPreloadRestartSelfTest; @@ -246,6 +247,7 @@ public static TestSuite suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, CacheLoadingConcurrentGridStartSelfTestAllowOverwrite.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheTxLoadingConcurrentGridStartSelfTestAllowOverwrite.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridPartitionedBackupLoadSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheGetReadFromBackupFailoverTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedLoadCacheSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedEventSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionNotLoadedEventSelfTest.class, ignoredTests); From 0825fa7ecbdb3c979541a351276ba9c5643054b2 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Sat, 1 Dec 2018 00:53:54 +0300 Subject: [PATCH 195/403] IGNITE-10427 Fix GridClusterStateProcessor#changeGlobalState0() should wrap future before sending ChangeGlobalStateMessage - Fixes #5513. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit 27b88d2b4d6efa91906585cc70772bd9c640950b) --- .../cluster/GridClusterStateProcessor.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index 353d4e36cbe47..d64f42b7f5296 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -905,15 +905,19 @@ private IgniteInternalFuture changeGlobalState0(final boolean activate, forceChangeBaselineTopology, System.currentTimeMillis()); + IgniteInternalFuture resFut = wrapStateChangeFuture(startedFut, msg); + try { if (log.isInfoEnabled()) U.log(log, "Sending " + prettyStr(activate) + " request with BaselineTopology " + blt); ctx.discovery().sendCustomEvent(msg); - if (ctx.isStopping()) - startedFut.onDone(new IgniteCheckedException("Failed to execute " + prettyStr(activate) + " request, " + - "node is stopping.")); + if (ctx.isStopping()) { + String errMsg = "Failed to execute " + prettyStr(activate) + " request, node is stopping."; + + startedFut.onDone(new IgniteCheckedException(errMsg)); + } } catch (IgniteCheckedException e) { U.error(log, "Failed to send global state change request: " + activate, e); @@ -921,7 +925,7 @@ private IgniteInternalFuture changeGlobalState0(final boolean activate, startedFut.onDone(e); } - return wrapStateChangeFuture(startedFut, msg); + return resFut; } /** {@inheritDoc} */ From 16674ac807bc78f8ce9ee4da34da98af9214773a Mon Sep 17 00:00:00 2001 From: Slava Koptilin Date: Sat, 1 Dec 2018 16:00:29 +0300 Subject: [PATCH 196/403] IGNITE-10413 Perform cache validation logic on primary node instead of near node - Fixes #5502. Signed-off-by: Ivan Rakov (cherry picked from commit d36055e) --- .../apache/ignite/internal/IgniteKernal.java | 9 ++-- .../ignite/internal/IgniteNodeAttributes.java | 3 ++ .../dht/GridDhtTxPrepareFuture.java | 36 ++++++++++++++++ .../dht/atomic/GridDhtAtomicCache.java | 41 +++++++++++++++++-- .../GridNearAtomicAbstractUpdateRequest.java | 4 +- ...OptimisticSerializableTxPrepareFuture.java | 3 +- .../GridNearOptimisticTxPrepareFuture.java | 3 +- .../GridNearPessimisticTxPrepareFuture.java | 3 +- .../near/GridNearTxPrepareRequest.java | 21 +++++++++- .../IgniteTxImplicitSingleStateImpl.java | 5 +++ .../transactions/IgniteTxLocalState.java | 5 +++ .../cache/transactions/IgniteTxStateImpl.java | 7 +++- 12 files changed, 126 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index b20023864ef07..59b16abd32b77 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -102,7 +102,6 @@ import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cluster.ClusterGroupAdapter; import org.apache.ignite.internal.cluster.IgniteClusterEx; -import org.apache.ignite.internal.managers.encryption.GridEncryptionManager; import org.apache.ignite.internal.managers.GridManager; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager; import org.apache.ignite.internal.managers.collision.GridCollisionManager; @@ -110,6 +109,7 @@ import org.apache.ignite.internal.managers.deployment.GridDeploymentManager; import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; +import org.apache.ignite.internal.managers.encryption.GridEncryptionManager; import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; import org.apache.ignite.internal.managers.failover.GridFailoverManager; import org.apache.ignite.internal.managers.indexing.GridIndexingManager; @@ -129,8 +129,8 @@ import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; import org.apache.ignite.internal.processors.cache.mvcc.MvccProcessorImpl; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; -import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.DataStorageMXBeanImpl; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; @@ -202,9 +202,9 @@ import org.apache.ignite.marshaller.MarshallerExclusions; import org.apache.ignite.marshaller.MarshallerUtils; import org.apache.ignite.marshaller.jdk.JdkMarshaller; -import org.apache.ignite.mxbean.FailureHandlingMxBean; import org.apache.ignite.mxbean.ClusterMetricsMXBean; import org.apache.ignite.mxbean.DataStorageMXBean; +import org.apache.ignite.mxbean.FailureHandlingMxBean; import org.apache.ignite.mxbean.IgniteMXBean; import org.apache.ignite.mxbean.StripedExecutorMXBean; import org.apache.ignite.mxbean.ThreadPoolMXBean; @@ -274,6 +274,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SPI_CLASS; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_VALIDATE_CACHE_REQUESTS; import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR; import static org.apache.ignite.internal.IgniteVersionUtils.BUILD_TSTAMP_STR; import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT; @@ -1565,6 +1566,8 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep add(ATTR_CONSISTENCY_CHECK_SKIPPED, getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK)); + add(ATTR_VALIDATE_CACHE_REQUESTS, Boolean.TRUE); + if (cfg.getConsistentId() != null) add(ATTR_NODE_CONSISTENT_ID, cfg.getConsistentId()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java index 4ca4f1b40585a..3945adf33d38d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java @@ -205,6 +205,9 @@ public final class IgniteNodeAttributes { /** Internal attribute name constant. */ public static final String ATTR_DYNAMIC_CACHE_START_ROLLBACK_SUPPORTED = ATTR_PREFIX + ".dynamic.cache.start.rollback.supported"; + /** Internal attribute indicates that incoming cache requests should be validated on primary node as well. */ + public static final String ATTR_VALIDATE_CACHE_REQUESTS = ATTR_CACHE + ".validate.cache.requests"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java index 6958ac5f7978b..4650b5ef81837 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java @@ -99,12 +99,14 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_LOADED; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_VALIDATE_CACHE_REQUESTS; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.NOOP; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE; +import static org.apache.ignite.internal.util.lang.GridFunc.isEmpty; import static org.apache.ignite.transactions.TransactionState.PREPARED; /** @@ -1052,6 +1054,24 @@ public void prepare(GridNearTxPrepareRequest req) { this.req = req; + ClusterNode node = cctx.discovery().node(tx.topologyVersion(), tx.nearNodeId()); + + boolean validateCache = needCacheValidation(node); + + if (validateCache) { + GridDhtTopologyFuture topFut = cctx.exchange().lastFinishedFuture(); + + if (topFut != null && !isEmpty(req.writes())) { + // All caches either read only or not. So validation of one cache context is enough. + GridCacheContext ctx = F.first(req.writes()).context(); + + Throwable err = topFut.validateCache(ctx, req.recovery(), isEmpty(req.writes()), null, null); + + if (err != null) + onDone(null, new IgniteCheckedException(err)); + } + } + boolean ser = tx.serializable() && tx.optimistic(); if (!F.isEmpty(req.writes()) || (ser && !F.isEmpty(req.reads()))) { @@ -1083,6 +1103,22 @@ public void prepare(GridNearTxPrepareRequest req) { mapIfLocked(); } + /** + * Returns {@code true} if cache validation needed. + * + * @param node Originating node. + * @return {@code True} if cache should be validated, {@code false} - otherwise. + */ + private boolean needCacheValidation(ClusterNode node) { + if (node == null) { + // The originating (aka near) node has left the topology + // and therefore the cache validation doesn't make sense. + return false; + } + + return Boolean.TRUE.equals(node.attribute(ATTR_VALIDATE_CACHE_REQUESTS)); + } + /** * Checks if this transaction needs previous value for the given tx entry. Will use passed in map to store * required key or will create new map if passed in map is {@code null}. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 09dc4dfe119f1..109c2c29e9c2b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -42,7 +42,6 @@ import org.apache.ignite.internal.UnregisteredClassException; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; -import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.affinity.AffinityAssignment; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryPredicate; @@ -69,12 +68,13 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedGetFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest; @@ -84,6 +84,7 @@ import org.apache.ignite.internal.processors.cache.dr.GridCacheDrExpirationInfo; import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext; @@ -120,6 +121,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_DEFERRED_ACK_TIMEOUT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_VALIDATE_CACHE_REQUESTS; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE; @@ -1773,11 +1775,31 @@ private void updateAllAsyncInternal0( if (!req.topologyLocked()) { // Can not wait for topology future since it will break // GridNearAtomicCheckUpdateRequest processing. - remap = !top.topologyVersionFuture().exchangeDone() || + remap = !top.topologyVersionFuture().isDone() || needRemap(req.topologyVersion(), top.readyTopologyVersion()); } if (!remap) { + boolean validateCache = needCacheValidation(node); + + if (validateCache) { + GridDhtTopologyFuture topFut = top.topologyVersionFuture(); + + assert topFut.isDone() : topFut; + + Throwable err = topFut.validateCache(ctx, req.recovery(), false, null, null); + + if (err != null) { + IgniteCheckedException e = new IgniteCheckedException(err); + + res.error(e); + + completionCb.apply(req, res); + + return; + } + } + update(node, locked, req, res, updDhtRes); dhtFut = updDhtRes.dhtFuture(); @@ -3638,6 +3660,17 @@ private void sendNearUpdateReply(UUID nodeId, GridNearAtomicUpdateResponse res) } } + /** + * Returns {@code true} if cache validation needed. + * + * @return {@code True} if cache should be validated, {@code false} - otherwise. + */ + private boolean needCacheValidation(ClusterNode node) { + assert node != null: "The near node must not be null. This is guaranteed by processNearAtomicUpdateRequest()"; + + return Boolean.TRUE.equals(node.attribute(ATTR_VALIDATE_CACHE_REQUESTS)); + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(GridDhtAtomicCache.class, this, super.toString()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java index 64fe1eef8fe6c..f0d89bf952a09 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java @@ -396,14 +396,14 @@ public void keepBinary(boolean val) { } /** - * @return Keep binary flag. + * @return Recovery flag. */ public final boolean recovery() { return isFlag(RECOVERY_FLAG_MASK); } /** - * @param val Keep binary flag. + * @param val Recovery flag. */ public void recovery(boolean val) { setFlag(val, RECOVERY_FLAG_MASK); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java index 30e6953baf233..87a414c3074b8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java @@ -554,7 +554,8 @@ private GridNearTxPrepareRequest createRequest( tx.taskNameHash(), m.clientFirst(), txNodes.size() == 1, - tx.activeCachesDeploymentEnabled()); + tx.activeCachesDeploymentEnabled(), + tx.txState().recovery()); for (IgniteTxEntry txEntry : writes) { if (txEntry.op() == TRANSFORM) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java index 5883b941148d8..c4e02844fa007 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java @@ -539,7 +539,8 @@ private void proceedPrepare(GridDistributedTxMapping m, @Nullable final Queue> txNod tx.taskNameHash(), false, true, - tx.activeCachesDeploymentEnabled()); + tx.activeCachesDeploymentEnabled(), + tx.txState().recovery()); req.queryUpdate(m.queryUpdate()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java index 7b2d26f64df62..00d969705011f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java @@ -58,6 +58,9 @@ public class GridNearTxPrepareRequest extends GridDistributedTxPrepareRequest { /** */ private static final int ALLOW_WAIT_TOP_FUT_FLAG_MASK = 0x10; + /** Recovery value flag. */ + private static final int RECOVERY_FLAG_MASK = 0x40; + /** Future ID. */ private IgniteUuid futId; @@ -122,7 +125,8 @@ public GridNearTxPrepareRequest( int taskNameHash, boolean firstClientReq, boolean allowWaitTopFut, - boolean addDepInfo + boolean addDepInfo, + boolean recovery ) { super(tx, timeout, @@ -147,6 +151,7 @@ public GridNearTxPrepareRequest( setFlag(explicitLock, EXPLICIT_LOCK_FLAG_MASK); setFlag(firstClientReq, FIRST_CLIENT_REQ_FLAG_MASK); setFlag(allowWaitTopFut, ALLOW_WAIT_TOP_FUT_FLAG_MASK); + setFlag(recovery, RECOVERY_FLAG_MASK); } /** @@ -157,6 +162,20 @@ public boolean allowWaitTopologyFuture() { return isFlag(ALLOW_WAIT_TOP_FUT_FLAG_MASK); } + /** + * @return Recovery flag. + */ + public final boolean recovery() { + return isFlag(RECOVERY_FLAG_MASK); + } + + /** + * @param val Recovery flag. + */ + public void recovery(boolean val) { + setFlag(val, RECOVERY_FLAG_MASK); + } + /** * @return {@code True} if first optimistic tx prepare request sent from client node. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java index 2b76772b7708a..808b7caa7f6bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java @@ -302,6 +302,11 @@ public class IgniteTxImplicitSingleStateImpl extends IgniteTxLocalStateAdapter { return ctx0 != null && ctx0.mvccEnabled(); } + /** {@inheritDoc} */ + @Override public boolean recovery() { + return recovery; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(IgniteTxImplicitSingleStateImpl.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java index 01eb4f4417877..e007f903b03ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java @@ -57,4 +57,9 @@ public interface IgniteTxLocalState extends IgniteTxState { * @param partId Partition id. */ public void touchPartition(int cacheId, int partId); + + /** + * @return Recovery mode flag. + */ + public boolean recovery(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java index cd8c6e9a87a2e..8a00244ae8eae 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java @@ -149,7 +149,7 @@ public class IgniteTxStateImpl extends IgniteTxLocalStateAdapter { assert ctx != null : cacheId; - Throwable err = topFut.validateCache(ctx, recovery != null && recovery, read, null, e.getValue()); + Throwable err = topFut.validateCache(ctx, recovery(), read, null, e.getValue()); if (err != null) { if (invalidCaches != null) @@ -180,6 +180,11 @@ public class IgniteTxStateImpl extends IgniteTxLocalStateAdapter { return null; } + /** {@inheritDoc} */ + @Override public boolean recovery() { + return recovery != null && recovery; + } + /** {@inheritDoc} */ @Override public CacheWriteSynchronizationMode syncMode(GridCacheSharedContext cctx) { CacheWriteSynchronizationMode syncMode = CacheWriteSynchronizationMode.FULL_ASYNC; From 167a517ff88284374af6c3ace9a5f25a1415c372 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Mon, 3 Dec 2018 11:02:51 +0300 Subject: [PATCH 197/403] IGNITE-10277 Fixed prepare-commit WAL ordering for one-phase commit - Fixes #5448. Signed-off-by: Alexey Goncharuk (cherry picked from commit 395811d) --- .../dht/GridDhtTransactionalCacheAdapter.java | 6 +++-- .../cache/distributed/dht/GridDhtTxLocal.java | 7 ++++- .../dht/GridDhtTxPrepareFuture.java | 19 ++++++++++--- ...OptimisticSerializableTxPrepareFuture.java | 11 +++++--- .../GridNearOptimisticTxPrepareFuture.java | 5 ++-- .../GridNearPessimisticTxPrepareFuture.java | 5 ++-- .../cache/transactions/IgniteTxAdapter.java | 27 +++++++++++++++++-- .../cache/transactions/IgniteTxHandler.java | 24 ++++++++++++++--- 8 files changed, 85 insertions(+), 19 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index 6a7a2a3080969..e1d636b45829a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -1121,7 +1121,8 @@ public IgniteInternalFuture lockAllAsync( req.txSize(), null, req.subjectId(), - req.taskNameHash()); + req.taskNameHash(), + null); if (req.syncCommit()) tx.syncMode(FULL_SYNC); @@ -2157,7 +2158,8 @@ public GridDhtTxLocal initTxTopologyVersion(UUID nodeId, -1, null, txSubjectId, - txTaskNameHash); + txTaskNameHash, + null); // if (req.syncCommit()) tx.syncMode(FULL_SYNC); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java index ca451f064eee3..ec92c402a288b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.distributed.GridCacheMappedVersion; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishResponse; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; @@ -119,6 +120,7 @@ public GridDhtTxLocal() { * @param storeEnabled Store enabled flag. * @param txSize Expected transaction size. * @param txNodes Transaction nodes mapping. + * @param parentTx Transaction from which this transaction was copied by(if it was). */ public GridDhtTxLocal( GridCacheSharedContext cctx, @@ -142,7 +144,8 @@ public GridDhtTxLocal( int txSize, Map> txNodes, UUID subjId, - int taskNameHash + int taskNameHash, + GridNearTxLocal parentTx ) { super( cctx, @@ -174,6 +177,8 @@ public GridDhtTxLocal( threadId = nearThreadId; + setParentTx(parentTx); + assert !F.eq(xidVer, nearXidVer); initResult(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java index 4650b5ef81837..a7816c5eebdba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java @@ -1010,7 +1010,7 @@ private boolean isMini(IgniteInternalFuture f) { * @return {@code True} if {@code done} flag was changed as a result of this call. */ private boolean onComplete(@Nullable GridNearTxPrepareResponse res) { - if ((last || tx.isSystemInvalidate()) && !(tx.near() && tx.local())) + if (!tx.onePhaseCommit() && ((last || tx.isSystemInvalidate()) && !(tx.near() && tx.local()))) tx.state(PREPARED); if (super.onDone(res, res == null ? err : null)) { @@ -1318,8 +1318,14 @@ private void prepare0() { if (isDone()) return; - if (last) + if (last) { + recheckOnePhaseCommit(); + + if (tx.onePhaseCommit()) + tx.chainState(PREPARED); + sendPrepareRequests(); + } } finally { markInitialized(); @@ -1327,9 +1333,9 @@ private void prepare0() { } /** - * + * Checking that one phase commit for transaction still actual. */ - private void sendPrepareRequests() { + private void recheckOnePhaseCommit() { if (tx.onePhaseCommit() && !tx.nearMap().isEmpty()) { for (GridDistributedTxMapping nearMapping : tx.nearMap().values()) { if (!tx.dhtMap().containsKey(nearMapping.primary().id())) { @@ -1339,7 +1345,12 @@ private void sendPrepareRequests() { } } } + } + /** + * + */ + private void sendPrepareRequests() { assert !tx.txState().mvccEnabled() || !tx.onePhaseCommit() || tx.mvccSnapshot() != null; int miniId = 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java index 87a414c3074b8..762e244d323b1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java @@ -270,7 +270,8 @@ private boolean isMini(IgniteInternalFuture f) { private boolean onComplete() { Throwable err0 = err; - if (err0 == null || tx.needCheckBackup()) + if ((!tx.onePhaseCommit() || tx.mappings().get(cctx.localNodeId()) == null) && + (err0 == null || tx.needCheckBackup())) tx.state(PREPARED); if (super.onDone(tx, err0)) { @@ -351,9 +352,13 @@ private void prepare( hasNearCache = true; } - for (IgniteTxEntry read : reads) + for (IgniteTxEntry read : reads) { map(read, topVer, mappings, txMapping, remap, topLocked); + if (read.context().isNear()) + hasNearCache = true; + } + if (keyLockFut != null) keyLockFut.onAllKeysAdded(); @@ -576,7 +581,7 @@ private void prepareLocal(GridNearTxPrepareRequest req, final MiniFuture fut, final boolean nearEntries) { IgniteInternalFuture prepFut = nearEntries ? - cctx.tm().txHandler().prepareNearTxLocal(req) : + cctx.tm().txHandler().prepareNearTxLocal(tx, req) : cctx.tm().txHandler().prepareColocatedTx(tx, req); prepFut.listen(new CI1>() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java index c4e02844fa007..7e8c6888c899f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java @@ -299,7 +299,8 @@ private boolean isMini(IgniteInternalFuture f) { private boolean onComplete() { Throwable err0 = err; - if (err0 == null || tx.needCheckBackup()) + if ((!tx.onePhaseCommit() || tx.mappings().get(cctx.localNodeId()) == null) && + (err0 == null || tx.needCheckBackup())) tx.state(PREPARED); if (super.onDone(tx, err0)) { @@ -569,7 +570,7 @@ private void proceedPrepare(GridDistributedTxMapping m, @Nullable final Queue prepFut = - m.hasNearCacheEntries() ? cctx.tm().txHandler().prepareNearTxLocal(req) + m.hasNearCacheEntries() ? cctx.tm().txHandler().prepareNearTxLocal(tx, req) : cctx.tm().txHandler().prepareColocatedTx(tx, req); prepFut.listen(new CI1>() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java index fbb8e7f8d3135..e5095fd92b54c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java @@ -256,7 +256,7 @@ private void prepareLocal(GridNearTxPrepareRequest req, add((IgniteInternalFuture)fut); IgniteInternalFuture prepFut = nearEntries ? - cctx.tm().txHandler().prepareNearTxLocal(req) : + cctx.tm().txHandler().prepareNearTxLocal(tx, req) : cctx.tm().txHandler().prepareColocatedTx(tx, req); prepFut.listen(new CI1>() { @@ -440,7 +440,8 @@ private void preparePessimistic() { err = this.err; - if (err == null || tx.needCheckBackup()) + if ((!tx.onePhaseCommit() || tx.mappings().get(cctx.localNodeId()) == null) && + (err == null || tx.needCheckBackup())) tx.state(PREPARED); if (super.onDone(tx, err)) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index 306c580f42d04..f747161c043aa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.cache.transactions; +import javax.cache.expiry.ExpiryPolicy; +import javax.cache.processor.EntryProcessor; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; @@ -37,8 +39,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import javax.cache.expiry.ExpiryPolicy; -import javax.cache.processor.EntryProcessor; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; @@ -288,6 +288,9 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement @GridToStringExclude private volatile TxCounters txCounters; + /** Transaction from which this transaction was copied by(if it was). */ + private GridNearTxLocal parentTx; + /** * Empty constructor required for {@link Externalizable}. */ @@ -406,6 +409,13 @@ protected IgniteTxAdapter( consistentIdMapper = new ConsistentIdMapper(cctx.discovery()); } + /** + * @param parentTx Transaction from which this transaction was copied by. + */ + public void setParentTx(GridNearTxLocal parentTx) { + this.parentTx = parentTx; + } + /** * @return Mvcc info. */ @@ -1048,6 +1058,19 @@ public void onePhaseCommit(boolean onePhaseCommit) { return state(state, false); } + /** + * Changing state for this transaction as well as chained(parent) transactions. + * + * @param state Transaction state. + * @return {@code True} if transition was valid, {@code false} otherwise. + */ + public boolean chainState(TransactionState state) { + if (parentTx != null) + parentTx.state(state); + + return state(state); + } + /** {@inheritDoc} */ @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor") @Override public IgniteInternalFuture finishFuture() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 6ea03fc78b2f5..7b83edcf19f56 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -342,14 +342,17 @@ private IgniteTxEntry unmarshal(@Nullable Collection entries) thr } /** + * @param originTx Transaction for copy. * @param req Request. * @return Prepare future. */ - public IgniteInternalFuture prepareNearTxLocal(final GridNearTxPrepareRequest req) { + public IgniteInternalFuture prepareNearTxLocal( + final GridNearTxLocal originTx, + final GridNearTxPrepareRequest req) { // Make sure not to provide Near entries to DHT cache. req.cloneEntries(); - return prepareNearTx(ctx.localNode(), req); + return prepareNearTx(originTx, ctx.localNode(), req); } /** @@ -360,6 +363,20 @@ public IgniteInternalFuture prepareNearTxLocal(final @Nullable private IgniteInternalFuture prepareNearTx( final ClusterNode nearNode, final GridNearTxPrepareRequest req + ) { + return prepareNearTx(null, nearNode, req); + } + + /** + * @param originTx Transaction for copy. + * @param nearNode Node that initiated transaction. + * @param req Near prepare request. + * @return Prepare future or {@code null} if need retry operation. + */ + @Nullable private IgniteInternalFuture prepareNearTx( + final GridNearTxLocal originTx, + final ClusterNode nearNode, + final GridNearTxPrepareRequest req ) { IgniteTxEntry firstEntry; @@ -508,7 +525,8 @@ public IgniteInternalFuture prepareNearTxLocal(final req.txSize(), req.transactionNodes(), req.subjectId(), - req.taskNameHash() + req.taskNameHash(), + originTx ); tx = ctx.tm().onCreated(null, tx); From d6f352820ca9031a597919c511d3bcae0f5bac6d Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 3 Dec 2018 16:21:46 +0700 Subject: [PATCH 198/403] IGNITE-9941 Fixed parsing of boolean parameters returned from nconf. (cherry picked from commit 1502494bca6e6d3059878949f23cae03b947eb24) --- modules/web-console/backend/app/settings.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js index 1c7504142095e..233566b1b76d8 100644 --- a/modules/web-console/backend/app/settings.js +++ b/modules/web-console/backend/app/settings.js @@ -53,6 +53,14 @@ module.exports = { const dfltHost = packaged ? '0.0.0.0' : '127.0.0.1'; const dfltPort = packaged ? 80 : 3000; + // We need this function because nconf() can return String or Boolean. + // And in JS we cannot compare String with Boolean. + const _isTrue = (confParam) => { + const v = nconf.get(confParam); + + return v === 'true' || v === true; + }; + return { agent: { dists: nconf.get('agent:dists') || dfltAgentDists @@ -62,7 +70,7 @@ module.exports = { host: nconf.get('server:host') || dfltHost, port: _normalizePort(nconf.get('server:port') || dfltPort), // eslint-disable-next-line eqeqeq - SSLOptions: nconf.get('server:ssl') == 'true' && { + SSLOptions: _isTrue('server:ssl') && { enable301Redirects: true, trustXFPHeader: true, key: fs.readFileSync(nconf.get('server:key')), @@ -70,7 +78,7 @@ module.exports = { passphrase: nconf.get('server:keyPassphrase') }, // eslint-disable-next-line eqeqeq - disableSignup: nconf.get('server:disable:signup') == 'true' + disableSignup: _isTrue('server:disable:signup') }, mail, mongoUrl: nconf.get('mongodb:url') || 'mongodb://127.0.0.1/console', From acc6ec3c817f365ca5bf3e9c99e04d3f6be00d6b Mon Sep 17 00:00:00 2001 From: devozerov Date: Mon, 3 Dec 2018 18:29:42 +0300 Subject: [PATCH 199/403] IGNITE-10291: SQL: fixed index re-build routine. Now indexes which appeared during node shutdown are rebuilt as well. This closes #5525. --- .../cache/IgniteCacheOffheapManager.java | 5 +- .../cache/IgniteCacheOffheapManagerImpl.java | 4 +- .../GridCacheDatabaseSharedManager.java | 14 +- .../persistence/GridCacheOffheapManager.java | 15 +- .../cache/persistence/IndexStorage.java | 26 ++- .../cache/persistence/IndexStorageImpl.java | 37 +++- .../UpgradePendingTreeToPerPartitionTask.java | 4 +- .../processors/query/GridQueryIndexing.java | 16 +- .../processors/query/GridQueryProcessor.java | 92 +++------ ...niteClientCacheInitializationFailTest.java | 10 +- .../database/IndexStorageSelfTest.java | 21 +- .../processors/query/h2/IgniteH2Indexing.java | 111 ++++++++--- .../query/h2/IndexBuildClosure.java | 54 ++++++ ...sure.java => IndexRebuildFullClosure.java} | 8 +- .../query/h2/IndexRebuildPartialClosure.java | 76 ++++++++ .../processors/query/h2/database/H2Tree.java | 13 ++ .../query/h2/database/H2TreeIndex.java | 47 +++-- .../processors/query/h2/opt/GridH2Table.java | 55 +++++- .../cache/IgniteDynamicSqlRestoreTest.java | 180 ++++++++++++++++-- .../query/h2/GridIndexRebuildSelfTest.java | 9 +- 20 files changed, 619 insertions(+), 178 deletions(-) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexBuildClosure.java rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/{RebuildIndexFromHashClosure.java => IndexRebuildFullClosure.java} (89%) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildPartialClosure.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index cdc44a55931b5..ed3697aac57b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -588,17 +588,18 @@ public long cacheEntriesCount(int cacheId, boolean primary, boolean backup, Affi /** * @param cacheId Cache ID. * @param idxName Index name. + * @param segment Segment. * @return Root page for index tree. * @throws IgniteCheckedException If failed. */ - public RootPage rootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException; + public RootPage rootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException; /** * @param cacheId Cache ID. * @param idxName Index name. * @throws IgniteCheckedException If failed. */ - public void dropRootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException; + public void dropRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException; /** * @param idxName Index name. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index c00dd1ace1408..8b1578fe9fc04 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -1143,14 +1143,14 @@ private long allocateForTree() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public RootPage rootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException { + @Override public RootPage rootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException { long pageId = allocateForTree(); return new RootPage(new FullPageId(pageId, grp.groupId()), true); } /** {@inheritDoc} */ - @Override public void dropRootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException { + @Override public void dropRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index d1dee32da9aac..5d82bee7a9ec4 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -136,6 +136,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.processors.port.GridPortRecord; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.util.GridMultiCollectionWrapper; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.future.CountDownFuture; @@ -1334,16 +1335,19 @@ private void prepareIndexRebuildFuture(int cacheId) { /** {@inheritDoc} */ @Override public void rebuildIndexesIfNeeded(GridDhtPartitionsExchangeFuture fut) { - if (cctx.kernalContext().query().moduleEnabled()) { + GridQueryProcessor qryProc = cctx.kernalContext().query(); + + if (qryProc.moduleEnabled()) { for (final GridCacheContext cacheCtx : (Collection)cctx.cacheContexts()) { if (cacheCtx.startTopologyVersion().equals(fut.initialVersion())) { final int cacheId = cacheCtx.cacheId(); final GridFutureAdapter usrFut = idxRebuildFuts.get(cacheId); - if (!cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode() - && cacheCtx.group().persistenceEnabled()) { - IgniteInternalFuture rebuildFut = cctx.kernalContext().query() - .rebuildIndexesFromHash(Collections.singleton(cacheCtx.cacheId())); + IgniteInternalFuture rebuildFut = qryProc.rebuildIndexesFromHash(cacheCtx); + + if (rebuildFut != null) { + log().info("Started indexes rebuilding for cache [name=" + cacheCtx.name() + + ", grpName=" + cacheCtx.group().name() + ']'); assert usrFut != null : "Missing user future for cache: " + cacheCtx.name(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index fb287bbcfb6a3..2dfb41f3d106e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -142,6 +142,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple ctx.wal(), globalRemoveId(), grp.groupId(), + grp.sharedGroup(), PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX, reuseList, @@ -835,19 +836,13 @@ public void destroyPartitionStore(int grpId, int partId) throws IgniteCheckedExc } /** {@inheritDoc} */ - @Override public RootPage rootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException { - if (grp.sharedGroup()) - idxName = Integer.toString(cacheId) + "_" + idxName; - - return indexStorage.getOrAllocateForTree(idxName); + @Override public RootPage rootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException { + return indexStorage.allocateCacheIndex(cacheId, idxName, segment); } /** {@inheritDoc} */ - @Override public void dropRootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException { - if (grp.sharedGroup()) - idxName = Integer.toString(cacheId) + "_" + idxName; - - indexStorage.dropRootPage(idxName); + @Override public void dropRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException { + indexStorage.dropCacheIndex(cacheId, idxName, segment); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java index 5141b04868f7c..295ff00f26311 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java @@ -23,6 +23,17 @@ * Meta store. */ public interface IndexStorage { + /** + * Allocate page for cache index. Index name will be masked if needed. + * + * @param cacheId Cache ID. + * @param idxName Index name. + * @param segment Segment. + * @return Root page. + * @throws IgniteCheckedException If failed. + */ + public RootPage allocateCacheIndex(Integer cacheId, String idxName, int segment) throws IgniteCheckedException; + /** * Get or allocate initial page for an index. * @@ -31,7 +42,18 @@ public interface IndexStorage { * was newly allocated, and rootId that is counter which increments each time new page allocated. * @throws IgniteCheckedException If failed. */ - public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException; + public RootPage allocateIndex(String idxName) throws IgniteCheckedException; + + /** + * Deallocate index page and remove from tree. + * + * @param cacheId Cache ID. + * @param idxName Index name. + * @param segment Segment. + * @return Root ID or -1 if no page was removed. + * @throws IgniteCheckedException If failed. + */ + public RootPage dropCacheIndex(Integer cacheId, String idxName, int segment) throws IgniteCheckedException; /** * Deallocate index page and remove from tree. @@ -40,7 +62,7 @@ public interface IndexStorage { * @return Root ID or -1 if no page was removed. * @throws IgniteCheckedException If failed. */ - public RootPage dropRootPage(String idxName) throws IgniteCheckedException; + public RootPage dropIndex(String idxName) throws IgniteCheckedException; /** * Destroy this meta store. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java index 6248765525866..b29553c6b8e2a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java @@ -61,6 +61,9 @@ public class IndexStorageImpl implements IndexStorage { /** Cache group ID. */ private final int grpId; + /** Whether group is shared. */ + private final boolean grpShared; + /** */ private final int allocPartId; @@ -76,6 +79,7 @@ public IndexStorageImpl( final IgniteWriteAheadLogManager wal, final AtomicLong globalRmvId, final int grpId, + boolean grpShared, final int allocPartId, final byte allocSpace, final ReuseList reuseList, @@ -86,6 +90,7 @@ public IndexStorageImpl( try { this.pageMem = pageMem; this.grpId = grpId; + this.grpShared = grpShared; this.allocPartId = allocPartId; this.allocSpace = allocSpace; this.reuseList = reuseList; @@ -99,7 +104,15 @@ public IndexStorageImpl( } /** {@inheritDoc} */ - @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException { + @Override public RootPage allocateCacheIndex(Integer cacheId, String idxName, int segment) + throws IgniteCheckedException { + String maskedIdxName = maskCacheIndexName(cacheId, idxName, segment); + + return allocateIndex(maskedIdxName); + } + + /** {@inheritDoc} */ + @Override public RootPage allocateIndex(String idxName) throws IgniteCheckedException { final MetaTree tree = metaTree; synchronized (this) { @@ -132,8 +145,15 @@ public IndexStorageImpl( } /** {@inheritDoc} */ - @Override public RootPage dropRootPage(final String idxName) + @Override public RootPage dropCacheIndex(Integer cacheId, String idxName, int segment) throws IgniteCheckedException { + String maskedIdxName = maskCacheIndexName(cacheId, idxName, segment); + + return dropIndex(maskedIdxName); + } + + /** {@inheritDoc} */ + @Override public RootPage dropIndex(final String idxName) throws IgniteCheckedException { byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8); final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0)); @@ -151,6 +171,19 @@ public IndexStorageImpl( metaTree.destroy(); } + /** + * Mask cache index name. + * + * @param idxName Index name. + * @return Masked name. + */ + private String maskCacheIndexName(Integer cacheId, String idxName, int segment) { + if (grpShared) + idxName = Integer.toString(cacheId) + "_" + idxName; + + return idxName + "%" + segment; + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/migration/UpgradePendingTreeToPerPartitionTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/migration/UpgradePendingTreeToPerPartitionTask.java index 6fa039dd09fc4..1cba69a7e60e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/migration/UpgradePendingTreeToPerPartitionTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/migration/UpgradePendingTreeToPerPartitionTask.java @@ -131,14 +131,14 @@ private void processCacheGroup(CacheGroupContext grp) throws IgniteCheckedExcept IndexStorage indexStorage = ((GridCacheOffheapManager)grp.offheap()).getIndexStorage(); //TODO: IGNITE-5874: replace with some check-method to avoid unnecessary page allocation. - RootPage pendingRootPage = indexStorage.getOrAllocateForTree(PENDING_ENTRIES_TREE_NAME); + RootPage pendingRootPage = indexStorage.allocateIndex(PENDING_ENTRIES_TREE_NAME); if (pendingRootPage.isAllocated()) { log.info("No pending tree found for cache group: [grpId=" + grp.groupId() + ", grpName=" + grp.name() + ']'); // Nothing to do here as just allocated tree is obviously empty. - indexStorage.dropRootPage(PENDING_ENTRIES_TREE_NAME); + indexStorage.dropIndex(PENDING_ENTRIES_TREE_NAME); return; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java index 7aa4021d7b5a0..c048d011d9691 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java @@ -29,6 +29,7 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.SqlQuery; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker; @@ -293,19 +294,12 @@ public void remove(GridCacheContext cctx, GridQueryTypeDescriptor type, CacheDat throws IgniteCheckedException; /** - * Rebuilds all indexes of given type from hash index. + * Rebuild indexes for the given cache if necessary. * - * @param cacheName Cache name. - * @throws IgniteCheckedException If failed. - */ - public void rebuildIndexesFromHash(String cacheName) throws IgniteCheckedException; - - /** - * Marks all indexes of given type for rebuild from hash index, making them unusable until rebuild finishes. - * - * @param cacheName Cache name. + * @param cctx Cache context. + * @return Future completed when index rebuild finished. */ - public void markForRebuildFromHash(String cacheName); + public IgniteInternalFuture rebuildIndexesFromHash(GridCacheContext cctx); /** * Returns backup filter. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index accccbb123924..4ed53b2e9c5d9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; @@ -93,8 +94,6 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet; import org.apache.ignite.internal.util.GridSpinBusyLock; -import org.apache.ignite.internal.util.future.GridCompoundFuture; -import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.lang.GridCloseableIterator; import org.apache.ignite.internal.util.lang.GridClosureException; import org.apache.ignite.internal.util.lang.IgniteOutClosureX; @@ -106,8 +105,6 @@ import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.util.worker.GridWorker; -import org.apache.ignite.internal.util.worker.GridWorkerFuture; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteInClosure; @@ -1752,78 +1749,45 @@ public boolean belongsToTable(GridCacheContext cctx, String expCacheName, String /** * Rebuilds indexes for provided caches from corresponding hash indexes. * - * @param cacheIds Cache IDs. + * @param cctx Cache context. * @return Future that will be completed when rebuilding is finished. */ - public IgniteInternalFuture rebuildIndexesFromHash(Set cacheIds) { - if (!busyLock.enterBusy()) - throw new IllegalStateException("Failed to rebuild indexes from hash (grid is stopping)."); - - // Because of alt type ids, there can be few entries in 'types' for a single cache. - // In order to avoid processing a cache more than once, let's track processed names. - Set processedCacheNames = new HashSet<>(); - - try { - GridCompoundFuture fut = new GridCompoundFuture(); - - for (Map.Entry e : types.entrySet()) { - if (cacheIds.contains(CU.cacheId(e.getKey().cacheName())) && - processedCacheNames.add(e.getKey().cacheName())) - fut.add(rebuildIndexesFromHash(e.getKey().cacheName(), e.getValue())); - } - - fut.markInitialized(); - - return fut; - } - finally { - busyLock.leaveBusy(); - } - } - - /** - * @param cacheName Cache name. - * @param desc Type descriptor. - * @return Future that will be completed when rebuilding of all indexes is finished. - */ - private IgniteInternalFuture rebuildIndexesFromHash(@Nullable final String cacheName, - @Nullable final QueryTypeDescriptorImpl desc) { + public IgniteInternalFuture rebuildIndexesFromHash(GridCacheContext cctx) { + // Indexing module is disabled, nothing to rebuild. if (idx == null) - return new GridFinishedFuture<>(new IgniteCheckedException("Indexing is disabled.")); - - if (desc == null) - return new GridFinishedFuture<>(); - - final GridWorkerFuture fut = new GridWorkerFuture<>(); + return null; - idx.markForRebuildFromHash(cacheName); + // No data on non-affinity nodes. + if (!cctx.affinityNode()) + return null; - GridWorker w = new GridWorker(ctx.igniteInstanceName(), "index-rebuild-worker", log) { - @Override protected void body() { - try { - idx.rebuildIndexesFromHash(cacheName); + // No indexes to rebuild when there are no QueryEntities. + if (!cctx.isQueryEnabled()) + return null; - fut.onDone(); - } - catch (Exception e) { - fut.onDone(e); - } - catch (Throwable e) { - U.error(log, "Failed to rebuild indexes for type [cache=" + cacheName + - ", name=" + desc.name() + ']', e); + // No need to rebuild if cache has no data. + boolean empty = true; - fut.onDone(e); + for (IgniteCacheOffheapManager.CacheDataStore store : cctx.offheap().cacheDataStores()) { + if (!store.isEmpty()) { + empty = false; - throw e; - } + break; } - }; + } - fut.setWorker(w); + if (empty) + return null; - ctx.getExecutorService().execute(w); + if (!busyLock.enterBusy()) + throw new IllegalStateException("Failed to rebuild indexes from hash (grid is stopping)."); - return fut; + try { + return idx.rebuildIndexesFromHash(cctx); + } + finally { + busyLock.leaveBusy(); + } } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java index b1df28e37f86b..cea591082d7f5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java @@ -41,6 +41,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker; @@ -345,13 +346,8 @@ private static class FailedIndexing implements GridQueryIndexing { } /** {@inheritDoc} */ - @Override public void rebuildIndexesFromHash(String cacheName) throws IgniteCheckedException { - // No-op - } - - /** {@inheritDoc} */ - @Override public void markForRebuildFromHash(String cacheName) { - // No-op + @Override public IgniteInternalFuture rebuildIndexesFromHash(GridCacheContext cctx) { + return null; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java index bbdcd383ed9de..de2161a79863c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java @@ -96,15 +96,24 @@ private void metaAllocation() throws Exception { IndexStorageImpl metaStore = storeMap.get(cacheId); if (metaStore == null) { - metaStore = new IndexStorageImpl(mem, null, new AtomicLong(), cacheId, - PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX, - null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true, - null); + metaStore = new IndexStorageImpl( + mem, + null, + new AtomicLong(), + cacheId, + false, + PageIdAllocator.INDEX_PARTITION, + PageMemory.FLAG_IDX, + null, + mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), + true, + null + ); storeMap.put(cacheId, metaStore); } - final RootPage rootPage = metaStore.getOrAllocateForTree(idxName); + final RootPage rootPage = metaStore.allocateIndex(idxName); assertTrue(rootPage.isAllocated()); @@ -118,7 +127,7 @@ private void metaAllocation() throws Exception { String idxName = entry.getKey(); FullPageId rootPageId = entry.getValue().pageId(); - final RootPage rootPage = storeMap.get(cacheId).getOrAllocateForTree(idxName); + final RootPage rootPage = storeMap.get(cacheId).allocateIndex(idxName); assertEquals("Invalid root page ID restored [cacheId=" + cacheId + ", idxName=" + idxName + ']', rootPageId, rootPage.pageId()); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index d3486aed79ee7..ae192c1d355d9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -61,6 +61,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryImpl; import org.apache.ignite.internal.processors.cache.CacheObjectUtils; @@ -79,7 +80,6 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.query.CacheQueryPartitionInfo; -import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; @@ -120,7 +120,6 @@ import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase; import org.apache.ignite.internal.processors.query.h2.opt.GridH2PlainRowFactory; import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias; @@ -171,6 +170,8 @@ import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.internal.util.worker.GridWorkerFuture; import org.apache.ignite.lang.IgniteBiClosure; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteFuture; @@ -891,15 +892,7 @@ private void addInitialUserIndex(String schemaName, H2TableDescriptor desc, Grid // Populate index with existing cache data. final GridH2RowDescriptor rowDesc = h2Tbl.rowDescriptor(); - SchemaIndexCacheVisitorClosure clo = new SchemaIndexCacheVisitorClosure() { - @Override public void apply(CacheDataRow row) throws IgniteCheckedException { - GridH2Row h2Row = rowDesc.createRow(row); - - h2Idx.putx(h2Row); - } - }; - - cacheVisitor.visit(clo); + cacheVisitor.visit(new IndexBuildClosure(rowDesc, h2Idx)); // At this point index is in consistent state, promote it through H2 SQL statement, so that cached // prepared statements are re-built. @@ -3135,33 +3128,97 @@ public ThreadLocalObjectPool.Reusable detach() { return reusableConnection; } + /** {@inheritDoc} */ + @Override public IgniteInternalFuture rebuildIndexesFromHash(GridCacheContext cctx) { + // No data in fresh in-memory cache. + if (!cctx.group().persistenceEnabled()) + return null; + + IgnitePageStoreManager pageStore = cctx.shared().pageStore(); + + assert pageStore != null; + + SchemaIndexCacheVisitorClosure clo; + + if (!pageStore.hasIndexStore(cctx.groupId())) { + // If there are no index store, rebuild all indexes. + clo = new IndexRebuildFullClosure(cctx.queries(), cctx.mvccEnabled()); + } + else { + // Otherwise iterate over tables looking for missing indexes. + IndexRebuildPartialClosure clo0 = new IndexRebuildPartialClosure(); + + for (H2TableDescriptor tblDesc : tables(cctx.name())) { + assert tblDesc.table() != null; + + tblDesc.table().collectIndexesForPartialRebuild(clo0); + } + + if (clo0.hasIndexes()) + clo = clo0; + else + return null; + } + + // Closure prepared, do rebuild. + final GridWorkerFuture fut = new GridWorkerFuture<>(); + + markIndexRebuild(cctx.name(), true); + + GridWorker worker = new GridWorker(ctx.igniteInstanceName(), "index-rebuild-worker-" + cctx.name(), log) { + @Override protected void body() { + try { + rebuildIndexesFromHash0(cctx, clo); + + markIndexRebuild(cctx.name(), false); + + fut.onDone(); + } + catch (Exception e) { + fut.onDone(e); + } + catch (Throwable e) { + U.error(log, "Failed to rebuild indexes for cache: " + cctx.name(), e); + + fut.onDone(e); + + throw e; + } + } + }; + + fut.setWorker(worker); + + ctx.getExecutorService().execute(worker); + + return fut; + } + /** - * Rebuild indexes from hash index. + * Do index rebuild. * - * @param cacheName Cache name. + * @param cctx Cache context. + * @param clo Closure. * @throws IgniteCheckedException If failed. */ - @Override public void rebuildIndexesFromHash(String cacheName) throws IgniteCheckedException { - int cacheId = CU.cacheId(cacheName); - - GridCacheContext cctx = ctx.cache().context().cacheContext(cacheId); - - final GridCacheQueryManager qryMgr = cctx.queries(); - + protected void rebuildIndexesFromHash0(GridCacheContext cctx, SchemaIndexCacheVisitorClosure clo) + throws IgniteCheckedException { SchemaIndexCacheVisitor visitor = new SchemaIndexCacheVisitorImpl(cctx); - visitor.visit(new RebuildIndexFromHashClosure(qryMgr, cctx.mvccEnabled())); - - for (H2TableDescriptor tblDesc : tables(cacheName)) - tblDesc.table().markRebuildFromHashInProgress(false); + visitor.visit(clo); } - /** {@inheritDoc} */ - @Override public void markForRebuildFromHash(String cacheName) { + /** + * Mark tables for index rebuild, so that their indexes are not used. + * + * @param cacheName Cache name. + * @param val Value. + */ + private void markIndexRebuild(String cacheName, boolean val) { for (H2TableDescriptor tblDesc : tables(cacheName)) { assert tblDesc.table() != null; - tblDesc.table().markRebuildFromHashInProgress(true); + tblDesc.table().markRebuildFromHashInProgress(val); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexBuildClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexBuildClosure.java new file mode 100644 index 0000000000000..8d1923f2a2a0c --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexBuildClosure.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2; + +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; +import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorClosure; + +/** + * Index build closure. + */ +public class IndexBuildClosure implements SchemaIndexCacheVisitorClosure { + /** Row descriptor. */ + private final GridH2RowDescriptor rowDesc; + + /** Index. */ + private final GridH2IndexBase idx; + + /** + * Constructor. + * + * @param rowDesc Row descriptor. + * @param idx Target index. + */ + public IndexBuildClosure(GridH2RowDescriptor rowDesc, GridH2IndexBase idx) { + this.rowDesc = rowDesc; + this.idx = idx; + } + + /** {@inheritDoc} */ + @Override public void apply(CacheDataRow row) throws IgniteCheckedException { + GridH2Row row0 = rowDesc.createRow(row); + + idx.putx(row0); + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/RebuildIndexFromHashClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildFullClosure.java similarity index 89% rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/RebuildIndexFromHashClosure.java rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildFullClosure.java index b635eaca8892b..8018839a6ce7b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/RebuildIndexFromHashClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildFullClosure.java @@ -22,8 +22,10 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager; import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorClosure; -/** */ -class RebuildIndexFromHashClosure implements SchemaIndexCacheVisitorClosure { +/** + * Closure to rebuild all indexes. + */ +public class IndexRebuildFullClosure implements SchemaIndexCacheVisitorClosure { /** */ private final GridCacheQueryManager qryMgr; @@ -34,7 +36,7 @@ class RebuildIndexFromHashClosure implements SchemaIndexCacheVisitorClosure { * @param qryMgr Query manager. * @param mvccEnabled MVCC status flag. */ - RebuildIndexFromHashClosure(GridCacheQueryManager qryMgr, boolean mvccEnabled) { + public IndexRebuildFullClosure(GridCacheQueryManager qryMgr, boolean mvccEnabled) { this.qryMgr = qryMgr; this.mvccEnabled = mvccEnabled; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildPartialClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildPartialClosure.java new file mode 100644 index 0000000000000..2672f0627cdb8 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IndexRebuildPartialClosure.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2; + +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; +import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorClosure; + +import java.util.Collection; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; + +/** + * Closure to rebuild some cache indexes. + */ +public class IndexRebuildPartialClosure implements SchemaIndexCacheVisitorClosure { + /** Indexes. */ + private final Map> tblIdxs = new IdentityHashMap<>(); + + /** {@inheritDoc} */ + @Override public void apply(CacheDataRow row) throws IgniteCheckedException { + assert hasIndexes(); + + for (Map.Entry> tblIdxEntry : tblIdxs.entrySet()) { + GridH2Table tbl = tblIdxEntry.getKey(); + + GridH2Row row0 = tbl.rowDescriptor().createRow(row); + + for (GridH2IndexBase idx : tblIdxEntry.getValue()) + idx.putx(row0); + } + } + + /** + * @param idx Index to be rebuilt. + */ + public void addIndex(GridH2Table tbl, GridH2IndexBase idx) { + Collection idxs = tblIdxs.get(tbl); + + if (idxs == null) { + idxs = Collections.newSetFromMap(new IdentityHashMap<>()); + + idxs.add(idx); + + tblIdxs.put(tbl, idxs); + } + + idxs.add(idx); + } + + /** + * @return {@code True} if there is at least one index to rebuild. + */ + public boolean hasIndexes() { + return !tblIdxs.isEmpty(); + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java index 4c6c0c065f58f..34b7fa39cf8b9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java @@ -108,6 +108,9 @@ public abstract class H2Tree extends BPlusTree { /** */ private final IgniteLogger log; + /** Whether index was created from scratch during owning node lifecycle. */ + private final boolean created; + /** * Constructor. * @@ -191,6 +194,8 @@ protected H2Tree( this.log = log; initTree(initNew, inlineSize); + + this.created = initNew; } /** @@ -507,6 +512,14 @@ private void inlineSizeRecomendation(SearchRow row) { */ public abstract int compareValues(Value v1, Value v2); + /** + * @return {@code True} if index was created during curren node's lifetime, {@code False} if it was restored from + * disk. + */ + public boolean created() { + return created; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(H2Tree.class, this, "super", super.toString()); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java index e9cca9ee0381e..18a3bbfd922bb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java @@ -88,6 +88,9 @@ public class H2TreeIndex extends GridH2IndexBase { /** */ private final String idxName; + /** Tree name. */ + private final String treeName; + /** */ private final IgniteLogger log; @@ -135,9 +138,7 @@ public H2TreeIndex( int typeId = cctx.binaryMarshaller() ? typeDesc.typeId() : typeDesc.valueClass().hashCode(); - String treeName = (tbl.rowDescriptor() == null ? "" : typeId + "_") + idxName; - - treeName = BPlusTree.treeName(treeName, "H2Tree"); + treeName = BPlusTree.treeName((tbl.rowDescriptor() == null ? "" : typeId + "_") + idxName, "H2Tree"); if (cctx.affinityNode()) { inlineIdxs = getAvailableInlineColumns(cols); @@ -151,8 +152,8 @@ public H2TreeIndex( for (int i = 0; i < segments.length; i++) { db.checkpointReadLock(); - try { - RootPage page = getMetaPage(treeName, i); + try { + RootPage page = getMetaPage(i); segments[i] = new H2Tree( treeName, @@ -196,6 +197,30 @@ public H2TreeIndex( initDistributedJoinMessaging(tbl); } + /** + * Check if index exists in store. + * + * @return {@code True} if exists. + */ + public boolean rebuildRequired() { + assert segments != null; + + for (int i = 0; i < segments.length; i++) { + try { + H2Tree segment = segments[i]; + + if (segment.created()) + return true; + } + catch (Exception e) { + throw new IgniteException("Failed to check index tree root page existence [cacheName=" + cctx.name() + + ", tblName=" + tblName + ", idxName=" + idxName + ", segment=" + i + ']'); + } + } + + return false; + } + /** * @param cols Columns array. * @return List of {@link InlineIndexHelper} objects. @@ -413,7 +438,7 @@ private List getAvailableInlineColumns(IndexColumn[] cols) { tree.destroy(); - dropMetaPage(tree.getName(), i); + dropMetaPage(i); } } } @@ -508,22 +533,20 @@ private int computeInlineSize(List inlineIdxs, int cfgInlineS } /** - * @param name Name. * @param segIdx Segment index. * @return RootPage for meta page. * @throws IgniteCheckedException If failed. */ - private RootPage getMetaPage(String name, int segIdx) throws IgniteCheckedException { - return cctx.offheap().rootPageForIndex(cctx.cacheId(), name + "%" + segIdx); + private RootPage getMetaPage(int segIdx) throws IgniteCheckedException { + return cctx.offheap().rootPageForIndex(cctx.cacheId(), treeName, segIdx); } /** - * @param name Name. * @param segIdx Segment index. * @throws IgniteCheckedException If failed. */ - private void dropMetaPage(String name, int segIdx) throws IgniteCheckedException { - cctx.offheap().dropRootPageForIndex(cctx.cacheId(), name + "%" + segIdx); + private void dropMetaPage(int segIdx) throws IgniteCheckedException { + cctx.offheap().dropRootPageForIndex(cctx.cacheId(), treeName, segIdx); } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index a612b637980ad..ca5c3e97de814 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -26,7 +26,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteInterruptedException; @@ -35,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.query.QueryTable; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryField; +import org.apache.ignite.internal.processors.query.h2.IndexRebuildPartialClosure; import org.apache.ignite.internal.processors.query.h2.database.H2RowFactory; import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; @@ -88,7 +88,7 @@ public class GridH2Table extends TableBase { private final Map tmpIdxs = new HashMap<>(); /** */ - private final ReadWriteLock lock; + private final ReentrantReadWriteLock lock; /** */ private boolean destroyed; @@ -126,6 +126,7 @@ public class GridH2Table extends TableBase { * @param idxsFactory Indexes factory. * @param cctx Cache context. */ + @SuppressWarnings("ConstantConditions") public GridH2Table(CreateTableData createTblData, GridH2RowDescriptor desc, H2RowFactory rowFactory, GridH2SystemIndexFactory idxsFactory, GridCacheContext cctx) { super(createTblData); @@ -174,11 +175,14 @@ public GridH2Table(CreateTableData createTblData, GridH2RowDescriptor desc, H2Ro assert idxs != null; List clones = new ArrayList<>(idxs.size()); + for (Index index : idxs) { Index clone = createDuplicateIndexIfNeeded(index); + if (clone != null) clones.add(clone); } + idxs.addAll(clones); boolean hasHashIndex = idxs.size() >= 2 && index(0).getIndexType().isHash(); @@ -551,12 +555,39 @@ private void addToIndex(GridH2IndexBase idx, GridH2Row row, GridH2Row prevRow) { } /** + * Collect indexes for rebuild. * + * @param clo Closure. + */ + public void collectIndexesForPartialRebuild(IndexRebuildPartialClosure clo) { + for (int i = sysIdxsCnt; i < idxs.size(); i++) { + Index idx = idxs.get(i); + + if (idx instanceof H2TreeIndex) { + H2TreeIndex idx0 = (H2TreeIndex)idx; + + if (idx0.rebuildRequired()) + clo.addIndex(this, idx0); + } + } + } + + /** + * Mark or unmark index rebuild state. */ public void markRebuildFromHashInProgress(boolean value) { assert !value || (idxs.size() >= 2 && index(1).getIndexType().isHash()) : "Table has no hash index."; rebuildFromHashInProgress = value; + + lock.writeLock().lock(); + + try { + incrementModificationCounter(); + } + finally { + lock.writeLock().unlock(); + } } /** @@ -637,7 +668,7 @@ private Index commitUserIndex(Session ses, String idxName) { if (cloneIdx != null) database.addSchemaObject(ses, cloneIdx); - setModified(); + incrementModificationCounter(); return idx; } @@ -941,7 +972,7 @@ public void addColumns(List cols, boolean ifNotExists) { desc.refreshMetadataFromTypeDescriptor(); - setModified(); + incrementModificationCounter(); } finally { unlock(true); @@ -949,9 +980,10 @@ public void addColumns(List cols, boolean ifNotExists) { } /** + * Drop columns. * - * @param cols - * @param ifExists + * @param cols Columns. + * @param ifExists If EXISTS flag. */ public void dropColumns(List cols, boolean ifExists) { assert !ifExists || cols.size() == 1; @@ -1003,7 +1035,7 @@ public void dropColumns(List cols, boolean ifExists) { ((GridH2IndexBase)idx).refreshColumnIds(); } - setModified(); + incrementModificationCounter(); } finally { unlock(true); @@ -1031,6 +1063,15 @@ public void dropColumns(List cols, boolean ifExists) { return columns; } + /** + * Increment modification counter to force recompilation of existing prepared statements. + */ + private void incrementModificationCounter() { + assert lock.isWriteLockedByCurrentThread(); + + setModified(); + } + /** * Set insert hack flag. * diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java index f7dc7b41ba6de..d009f33f24a6b 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java @@ -18,12 +18,20 @@ package org.apache.ignite.internal.processors.cache; import java.io.Serializable; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Timestamp; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.Ignition; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.SqlFieldsQuery; @@ -33,8 +41,10 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; @@ -44,6 +54,7 @@ /** * */ +@SuppressWarnings("Duplicates") public class IgniteDynamicSqlRestoreTest extends GridCommonAbstractTest implements Serializable { public static final String TEST_CACHE_NAME = "test"; @@ -117,7 +128,7 @@ public void testMergeChangedConfigOnCoordinator() throws Exception { //and: change data try (IgniteDataStreamer s = ig.dataStreamer(TEST_CACHE_NAME)) { s.allowOverwrite(true); - for (int i = 0; i < 5_000; i++) + for (int i = 0; i < 50; i++) s.addData(i, null); } @@ -134,7 +145,7 @@ public void testMergeChangedConfigOnCoordinator() throws Exception { //then: everything is ok try (IgniteDataStreamer s = ig1.dataStreamer(TEST_CACHE_NAME)) { s.allowOverwrite(true); - for (int i = 0; i < 50_000; i++) { + for (int i = 0; i < 50; i++) { BinaryObject bo = ig1.binary().builder(TEST_INDEX_OBJECT) .setField("a", i, Object.class) .setField("b", String.valueOf(i), Object.class) @@ -147,11 +158,131 @@ public void testMergeChangedConfigOnCoordinator() throws Exception { IgniteCache cache = ig1.cache(TEST_CACHE_NAME); - assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa")); + assertIndexUsed(cache, "explain select * from TestIndexObject where a > 5", "myindexa"); assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty()); } } + /** + * @throws Exception If failed. + */ + @SuppressWarnings("AssertWithSideEffects") + public void testIndexCreationWhenNodeStopped() throws Exception { + // Start topology. + startGrid(0); + Ignite srv2 = startGrid(1); + Ignite cli; + + Ignition.setClientMode(true); + + try { + cli = startGrid(2); + } + finally { + Ignition.setClientMode(false); + } + + cli.cluster().active(true); + + // Create table, add some data. + int entryCnt = 50; + + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:10802")) { + executeJdbc(conn, + " CREATE TABLE PERSON (\n" + + " FIRST_NAME VARCHAR,\n" + + " LAST_NAME VARCHAR,\n" + + " ADDRESS VARCHAR,\n" + + " LANG VARCHAR,\n" + + " BIRTH_DATE TIMESTAMP,\n" + + " CONSTRAINT PK_PESON PRIMARY KEY (FIRST_NAME,LAST_NAME,ADDRESS,LANG)\n" + + " ) WITH \"key_type=PersonKeyType, CACHE_NAME=PersonCache, value_type=PersonValueType, AFFINITY_KEY=FIRST_NAME,template=PARTITIONED,backups=1\""); + + try (PreparedStatement stmt = conn.prepareStatement( + "insert into Person(LANG, FIRST_NAME, ADDRESS, LAST_NAME, BIRTH_DATE) values(?,?,?,?,?)")) { + for (int i = 0; i < entryCnt; i++) { + String s = String.valueOf(i); + + stmt.setString(1, s); + stmt.setString(2, s); + stmt.setString(3, s); + stmt.setString(4, s); + stmt.setTimestamp(5, new Timestamp(System.currentTimeMillis())); + + stmt.executeUpdate(); + } + } + } + + // Stop second node. + srv2.close(); + + // Create an index on remaining node. + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:10802")) { + executeJdbc(conn, "create index PERSON_FIRST_NAME_IDX on PERSON(FIRST_NAME)"); + } + + // Restart second node. + startGrid(1); + + // Await for index rebuild on started node. + assert GridTestUtils.waitForCondition(new PA() { + @Override public boolean apply() { + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:10801")) { + try (PreparedStatement stmt = conn.prepareStatement( + "EXPLAIN SELECT * FROM Person USE INDEX(PERSON_FIRST_NAME_IDX) WHERE FIRST_NAME=?")) { + stmt.setString(1, String.valueOf(1)); + + StringBuilder fullPlan = new StringBuilder(); + + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) + fullPlan.append(rs.getString(1)).append("; "); + } + + System.out.println("PLAN: " + fullPlan); + + return fullPlan.toString().contains("PUBLIC.PERSON_FIRST_NAME_IDX"); + } + } + catch (Exception e) { + throw new RuntimeException("Query failed.", e); + } + } + }, 5_000); + + // Make sure that data could be queried. + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:10802")) { + try (PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM Person USE INDEX(PERSON_FIRST_NAME_IDX) WHERE FIRST_NAME=?")) { + for (int i = 0; i < entryCnt; i ++) { + stmt.setString(1, String.valueOf(i)); + + try (ResultSet rs = stmt.executeQuery()) { + rs.next(); + + long cnt = rs.getLong(1); + + assertEquals(1L, cnt); + } + } + } + } + } + + /** + * Execute a statement through JDBC connection. + * + * @param conn Connection. + * @param sql Statement. + * @throws Exception If failed. + */ + private static void executeJdbc(Connection conn, String sql) throws Exception { + try (Statement stmt = conn.createStatement()) { + stmt.execute(sql); + } + } + /** * @throws Exception if failed. */ @@ -186,7 +317,7 @@ public void testTakeConfigFromJoiningNodeOnInactiveGrid() throws Exception { //then: config for cache was applying successful IgniteCache cache = ig.cache(TEST_CACHE_NAME); - assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa")); + assertIndexUsed(cache, "explain select * from TestIndexObject where a > 5", "myindexa"); assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty()); } } @@ -233,7 +364,7 @@ public void testResaveConfigAfterMerge() throws Exception { IgniteCache cache = ig.cache(TEST_CACHE_NAME); - assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa")); + assertIndexUsed(cache, "explain select * from TestIndexObject where a > 5", "myindexa"); assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty()); } } @@ -241,6 +372,7 @@ public void testResaveConfigAfterMerge() throws Exception { /** * @throws Exception if failed. */ + @SuppressWarnings("ArraysAsListWithZeroOrOneArgument") public void testMergeChangedConfigOnInactiveGrid() throws Exception { { //given: two started nodes with test table @@ -288,7 +420,7 @@ public void testMergeChangedConfigOnInactiveGrid() throws Exception { //then: config should be merged try (IgniteDataStreamer s = ig1.dataStreamer(TEST_CACHE_NAME)) { s.allowOverwrite(true); - for (int i = 0; i < 5_000; i++) { + for (int i = 0; i < 50; i++) { BinaryObject bo = ig1.binary().builder("TestIndexObject") .setField("a", i, Object.class) .setField("b", String.valueOf(i), Object.class) @@ -300,12 +432,31 @@ public void testMergeChangedConfigOnInactiveGrid() throws Exception { IgniteCache cache = ig1.cache(TEST_CACHE_NAME); //then: index "myindexa" and column "b" restored from node "1" - assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa")); - assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where b > 5"), containsString("myindexb")); + assertIndexUsed(cache, "explain select * from TestIndexObject where a > 5", "myindexa"); + assertIndexUsed(cache, "explain select * from TestIndexObject where b > 5", "myindexb"); assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b FROM TestIndexObject limit 1")).getAll().isEmpty()); } } + /** + * Make sure that index is used for the given statement. + * + * @param cache Cache. + * @param sql Statement. + * @param idx Index. + * @throws IgniteCheckedException If failed. + */ + private void assertIndexUsed(IgniteCache cache, String sql, String idx) + throws IgniteCheckedException { + assert GridTestUtils.waitForCondition(new PA() { + @Override public boolean apply() { + String plan = doExplainPlan(cache, sql); + + return plan.contains(idx); + } + }, 10_000); + } + /** * @throws Exception if failed. */ @@ -341,7 +492,7 @@ public void testTakeChangedConfigOnActiveGrid() throws Exception { //then: config should be merged try (IgniteDataStreamer s = ig.dataStreamer(TEST_CACHE_NAME)) { s.allowOverwrite(true); - for (int i = 0; i < 5_000; i++) { + for (int i = 0; i < 50; i++) { BinaryObject bo = ig.binary().builder("TestIndexObject") .setField("a", i, Object.class) .setField("b", String.valueOf(i), Object.class) @@ -355,7 +506,7 @@ public void testTakeChangedConfigOnActiveGrid() throws Exception { cache.indexReadyFuture().get(); - assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa")); + assertIndexUsed(cache, "explain select * from TestIndexObject where a > 5", "myindexa"); assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty()); } } @@ -363,6 +514,7 @@ public void testTakeChangedConfigOnActiveGrid() throws Exception { /** * @throws Exception if failed. */ + @SuppressWarnings("ConstantConditions") public void testFailJoiningNodeBecauseDifferentSql() throws Exception { { //given: two started nodes with test table @@ -407,6 +559,7 @@ public void testFailJoiningNodeBecauseDifferentSql() throws Exception { /** * @throws Exception if failed. */ + @SuppressWarnings("ConstantConditions") public void testFailJoiningNodeBecauseFieldInlineSizeIsDifferent() throws Exception { { //given: two started nodes with test table @@ -417,13 +570,13 @@ public void testFailJoiningNodeBecauseFieldInlineSizeIsDifferent() throws Except IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration()); - cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a) INLINE_SIZE 1000")).getAll(); + cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a) INLINE_SIZE 100")).getAll(); //stop one node and create index on other node stopGrid(1); cache.query(new SqlFieldsQuery("drop index myindexa")).getAll(); - cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a) INLINE_SIZE 2000")).getAll(); + cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a) INLINE_SIZE 200")).getAll(); //and: stopped all grid stopAllGrids(); @@ -447,6 +600,7 @@ public void testFailJoiningNodeBecauseFieldInlineSizeIsDifferent() throws Except /** * @throws Exception if failed. */ + @SuppressWarnings("ConstantConditions") public void testFailJoiningNodeBecauseNeedConfigUpdateOnActiveGrid() throws Exception { { startGrid(0); @@ -497,7 +651,7 @@ public void testFailJoiningNodeBecauseNeedConfigUpdateOnActiveGrid() throws Exce */ private void fillTestData(Ignite ig) { try (IgniteDataStreamer s = ig.dataStreamer(TEST_CACHE_NAME)) { - for (int i = 0; i < 50_000; i++) { + for (int i = 0; i < 500; i++) { BinaryObject bo = ig.binary().builder("TestIndexObject") .setField("a", i, Object.class) .setField("b", String.valueOf(i), Object.class) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java index c5f1441af5d69..d8ba0601287e7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java @@ -24,12 +24,14 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.index.DynamicIndexAbstractSelfTest; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorClosure; import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.typedef.internal.U; @@ -38,7 +40,7 @@ */ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest { /** Data size. */ - protected static final int AMOUNT = 300; + protected static final int AMOUNT = 50; /** Data size. */ protected static final String CACHE_NAME = "T"; @@ -228,13 +230,14 @@ private static class BlockingIndexing extends IgniteH2Indexing { private boolean firstRbld = true; /** {@inheritDoc} */ - @Override public void rebuildIndexesFromHash(String cacheName) throws IgniteCheckedException { + @Override protected void rebuildIndexesFromHash0(GridCacheContext cctx, SchemaIndexCacheVisitorClosure clo) + throws IgniteCheckedException { if (!firstRbld) U.await(INSTANCE.rebuildLatch); else firstRbld = false; - super.rebuildIndexesFromHash(cacheName); + super.rebuildIndexesFromHash0(cctx, clo); } } } From ee99929fd41677cab0310f37eae9511cf66bb81c Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Fri, 30 Nov 2018 20:16:19 +0300 Subject: [PATCH 200/403] IGNITE-10172 Added a system property to disable cache metrics sending across discovery - Fixes #5416. Signed-off-by: Alexey Goncharuk --- .../apache/ignite/IgniteSystemProperties.java | 10 ++ .../discovery/GridDiscoveryManager.java | 8 ++ .../CacheMetricsForClusterGroupSelfTest.java | 101 +++++++++++++++--- 3 files changed, 106 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 5d441e601067a..c5df7b5a07004 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1041,6 +1041,16 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_RECOVERY_VERBOSE_LOGGING = "IGNITE_RECOVERY_VERBOSE_LOGGING"; + /** + * When set to {@code true}, cache metrics are not included into the discovery metrics update message (in this + * case message contains only cluster metrics). By default cache metrics are included into the message and + * calculated each time the message is sent. + *

+ * Cache metrics sending can also be turned off by disabling statistics per each cache, but in this case some cache + * metrics will be unavailable via JMX too. + */ + public static final String IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE = "IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index e04b8d704daf0..f0b13c07aecf3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -48,6 +48,7 @@ import org.apache.ignite.IgniteClientDisconnectedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cluster.BaselineNode; @@ -1095,6 +1096,10 @@ private GridLocalMetrics createMetrics() { */ public DiscoveryMetricsProvider createMetricsProvider() { return new DiscoveryMetricsProvider() { + /** Disable cache metrics update. */ + private final boolean disableCacheMetricsUpdate = IgniteSystemProperties.getBoolean( + IgniteSystemProperties.IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE, false); + /** */ private final long startTime = U.currentTimeMillis(); @@ -1106,6 +1111,9 @@ public DiscoveryMetricsProvider createMetricsProvider() { /** {@inheritDoc} */ @Override public Map cacheMetrics() { try { + if (disableCacheMetricsUpdate) + return Collections.emptyMap(); + /** Caches should not be accessed while state transition is in progress. */ if (ctx.state().clusterState().transition()) return Collections.emptyMap(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java index 3f196a57ad80b..df60c42e8d4c6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; @@ -71,22 +72,15 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest return cfg; } - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - startGrids(GRID_CNT); - - daemon = true; - - startGrid(GRID_CNT); - } - /** * Test cluster group metrics in case of statistics enabled. */ public void testMetricsStatisticsEnabled() throws Exception { - createCaches(true); + startGrids(); try { + createCaches(true); + populateCacheData(cache1, ENTRY_CNT_CACHE1); populateCacheData(cache2, ENTRY_CNT_CACHE2); @@ -107,7 +101,7 @@ public void testMetricsStatisticsEnabled() throws Exception { assertMetrics(cache2, true); } finally { - destroyCaches(); + stopAllGrids(); } } @@ -117,9 +111,11 @@ public void testMetricsStatisticsEnabled() throws Exception { * @throws Exception If failed. */ public void testMetricsStatisticsDisabled() throws Exception { - createCaches(false); + startGrids(); try { + createCaches(false); + populateCacheData(cache1, ENTRY_CNT_CACHE1); populateCacheData(cache2, ENTRY_CNT_CACHE2); @@ -140,10 +136,61 @@ public void testMetricsStatisticsDisabled() throws Exception { assertMetrics(cache2, false); } finally { - destroyCaches(); + stopAllGrids(); + } + } + + /** + * Tests that only local metrics are updating if discovery updates disabled. + */ + public void testMetricsDiscoveryUpdatesDisabled() throws Exception { + System.setProperty(IgniteSystemProperties.IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE, "true"); + + try { + startGrids(); + + try { + createCaches(true); + + populateCacheData(cache1, ENTRY_CNT_CACHE1); + populateCacheData(cache2, ENTRY_CNT_CACHE2); + + readCacheData(cache1, ENTRY_CNT_CACHE1); + readCacheData(cache2, ENTRY_CNT_CACHE2); + + awaitMetricsUpdate(); + + Collection nodes = grid(0).cluster().forRemotes().nodes(); + + for (ClusterNode node : nodes) { + Map metrics = ((IgniteClusterNode)node).cacheMetrics(); + assertNotNull(metrics); + assertTrue(metrics.isEmpty()); + } + + assertOnlyLocalMetricsUpdating(CACHE1); + assertOnlyLocalMetricsUpdating(CACHE2); + } + finally { + stopAllGrids(); + } + } + finally { + System.setProperty(IgniteSystemProperties.IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE, "false"); } } + /** + * Start grids. + */ + private void startGrids() throws Exception { + startGrids(GRID_CNT); + + daemon = true; + + startGrid(GRID_CNT); + } + /** * @param statisticsEnabled Statistics enabled. */ @@ -267,6 +314,34 @@ private void assertMetrics(IgniteCache cache, boolean expectNo } } + /** + * Asserts that only local metrics are updating. + * + * @param cacheName Cache name. + */ + private void assertOnlyLocalMetricsUpdating(String cacheName) { + for (int i = 0; i < GRID_CNT; i++) { + IgniteCache cache = grid(i).cache(cacheName); + + CacheMetrics clusterMetrics = cache.metrics(grid(i).cluster().forCacheNodes(cacheName)); + CacheMetrics locMetrics = cache.localMetrics(); + + assertEquals(clusterMetrics.name(), locMetrics.name()); + + assertEquals(0L, clusterMetrics.getCacheGets()); + assertEquals(0L, cache.mxBean().getCacheGets()); + assertEquals(locMetrics.getCacheGets(), cache.localMxBean().getCacheGets()); + + assertEquals(0L, clusterMetrics.getCachePuts()); + assertEquals(0L, cache.mxBean().getCachePuts()); + assertEquals(locMetrics.getCachePuts(), cache.localMxBean().getCachePuts()); + + assertEquals(0L, clusterMetrics.getCacheHits()); + assertEquals(0L, cache.mxBean().getCacheHits()); + assertEquals(locMetrics.getCacheHits(), cache.localMxBean().getCacheHits()); + } + } + /** * @param ms Milliseconds. * @param f Function. From 330ee003f98593421399b9c1a999a07cc38745f6 Mon Sep 17 00:00:00 2001 From: ibessonov Date: Tue, 4 Dec 2018 12:55:45 +0300 Subject: [PATCH 201/403] IGNITE-10369 Avoid race on handling discovery messages if last custom message is not processed completely - Fixes #5462. Signed-off-by: Pavel Kovalenko (cherry picked from commit 0cd303f1f1b4be23be8f3be6301d4defd4be6ea1) --- .../discovery/GridDiscoveryManager.java | 4 +- .../cluster/GridClusterStateProcessor.java | 2 +- .../spi/discovery/DiscoveryDataBag.java | 6 +++ .../ignite/spi/discovery/tcp/ServerImpl.java | 41 ++++++++++++++++++- .../IgniteClusterActivateDeactivateTest.java | 4 +- .../zk/internal/ZookeeperDiscoveryImpl.java | 21 ++++++++++ 6 files changed, 70 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index f0b13c07aecf3..75558380ea226 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -767,8 +767,6 @@ else if (customMsg instanceof ChangeGlobalStateMessage) { // Current version. discoCache = discoCache(); - final DiscoCache discoCache0 = discoCache; - // If this is a local join event, just save it and do not notify listeners. if (locJoinEvt) { if (gridStartTime == 0) @@ -856,7 +854,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) { try { fut.get(); - discoWrk.addEvent(type, nextTopVer, node, discoCache0, topSnapshot, null); + discoWrk.addEvent(EVT_CLIENT_NODE_RECONNECTED, nextTopVer, node, discoCache, topSnapshot, null); } catch (IgniteException ignore) { // No-op. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index d64f42b7f5296..778d9f24e8631 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -415,7 +415,7 @@ private boolean isBaselineSatisfied(BaselineTopology blt, List serv if (msg.requestId().equals(state.transitionRequestId())) { log.info("Received state change finish message: " + msg.clusterActive()); - globalState = globalState.finish(msg.success()); + globalState = state.finish(msg.success()); afterStateChangeFinished(msg.id(), msg.success()); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java index b8d8908215130..2b6bbca0c5358 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.internal.GridComponent; +import org.apache.ignite.internal.util.typedef.internal.S; import org.jetbrains.annotations.Nullable; /** @@ -312,4 +313,9 @@ public Map commonData() { @Nullable public Map localNodeSpecificData() { return nodeSpecificData.get(DEFAULT_KEY); } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(DiscoveryDataBag.class, this); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index e6fd6d62e575a..fa9033d626fed 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -263,6 +263,9 @@ class ServerImpl extends TcpDiscoveryImpl { private final ConcurrentMap>> pingMap = new ConcurrentHashMap<>(); + /** Last listener future. */ + private IgniteFuture lastCustomEvtLsnrFut; + /** * @param adapter Adapter. */ @@ -2159,6 +2162,20 @@ private static WorkersRegistry getWorkerRegistry(TcpDiscoverySpi spi) { return spi.ignite() instanceof IgniteEx ? ((IgniteEx)spi.ignite()).context().workersRegistry() : null; } + /** + * Wait for all the listeners from previous discovery message to be completed. + */ + private void waitForLastCustomEventListenerFuture() { + if (lastCustomEvtLsnrFut != null) { + try { + lastCustomEvtLsnrFut.get(); + } + finally { + lastCustomEvtLsnrFut = null; + } + } + } + /** * Discovery messages history used for client reconnect. */ @@ -4202,6 +4219,15 @@ private void trySendMessageDirectly(TcpDiscoveryNode node, TcpDiscoveryAbstractM private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) { assert msg != null; + blockingSectionBegin(); + + try { + waitForLastCustomEventListenerFuture(); + } + finally { + blockingSectionEnd(); + } + TcpDiscoveryNode node = msg.node(); assert node != null; @@ -5649,8 +5675,19 @@ private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg, boolean hist, msgObj); - if (waitForNotification || msgObj.isMutable()) - fut.get(); + if (waitForNotification || msgObj.isMutable()) { + blockingSectionBegin(); + + try { + fut.get(); + } + finally { + blockingSectionEnd(); + } + } + else { + lastCustomEvtLsnrFut = fut; + } if (msgObj.isMutable()) { try { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java index f2682376115a6..1949829d003c8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -1203,13 +1203,13 @@ private void stateChangeFailover3(boolean activate) throws Exception { client = false; - IgniteInternalFuture startFut1 = GridTestUtils.runAsync((Callable)() -> { + IgniteInternalFuture startFut1 = GridTestUtils.runAsync(() -> { startGrid(4); return null; }, "start-node1"); - IgniteInternalFuture startFut2 = GridTestUtils.runAsync((Callable)() -> { + IgniteInternalFuture startFut2 = GridTestUtils.runAsync(() -> { startGrid(5); return null; diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java index 3d42042b850fb..2462ae60d2cdd 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java @@ -192,6 +192,9 @@ public class ZookeeperDiscoveryImpl { /** */ private final ZookeeperDiscoveryStatistics stats; + /** Last listener future. */ + private IgniteFuture lastCustomEvtLsnrFut; + /** * @param spi Discovery SPI. * @param igniteInstanceName Instance name. @@ -2754,6 +2757,8 @@ private void processNewEvents(final ZkDiscoveryEventsData evtsData) throws Excep private boolean processBulkJoin(ZkDiscoveryEventsData evtsData, ZkDiscoveryNodeJoinEventData evtData) throws Exception { + waitForLastListenerFuture(); + boolean evtProcessed = false; for (int i = 0; i < evtData.joinedNodes.size(); i++) { @@ -3434,6 +3439,8 @@ private void notifyCustomEvent(final ZkDiscoveryCustomEventData evtData, final D if (msg != null && msg.isMutable()) fut.get(); + else + lastCustomEvtLsnrFut = fut; } /** @@ -3991,6 +3998,20 @@ public static byte[] unzip(byte[] zipBytes) throws DataFormatException { return out.toByteArray(); } + /** + * Wait for all the listeners from previous discovery message to be completed. + */ + private void waitForLastListenerFuture() { + if (lastCustomEvtLsnrFut != null) { + try { + lastCustomEvtLsnrFut.get(); + } + finally { + lastCustomEvtLsnrFut = null; + } + } + } + /** * */ From ae3575e69b1925bc8e608badd8836e583d5366f3 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Tue, 4 Dec 2018 17:40:04 +0300 Subject: [PATCH 202/403] IGNITE-10457: MVCC TX: Fix MvccProcessor initialization on recovery from store. This closes #5538. (cherry picked from commit cdba7d9a1a1d0e2f69aafaeeef3a2746bc69b072) --- .../processors/cache/GridCacheProcessor.java | 11 +++++---- .../cache/mvcc/MvccProcessorImpl.java | 24 ++++++++++++------- .../DatabaseLifecycleListener.java | 16 ++++++++----- .../GridCacheDatabaseSharedManager.java | 8 ++++--- .../testframework/MvccFeatureChecker.java | 2 +- .../query/h2/GridIndexRebuildSelfTest.java | 2 +- 6 files changed, 39 insertions(+), 24 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index ed021ba39408d..b69df8da44916 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache; -import javax.management.MBeanServer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -5477,7 +5476,9 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da } /** {@inheritDoc} */ - @Override public void afterBinaryMemoryRestore(GridCacheDatabaseSharedManager.RestoreBinaryState binaryState) throws IgniteCheckedException { + @Override public void afterBinaryMemoryRestore( + IgniteCacheDatabaseSharedManager mgr, + GridCacheDatabaseSharedManager.RestoreBinaryState restoreState) throws IgniteCheckedException { for (DynamicCacheDescriptor cacheDescriptor : persistentCaches()) { // Skip MVCC caches. // TODO: https://issues.apache.org/jira/browse/IGNITE-10052 @@ -5491,8 +5492,10 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da } /** {@inheritDoc} */ - @Override public void afterLogicalUpdatesApplied(GridCacheDatabaseSharedManager.RestoreLogicalState logicalState) throws IgniteCheckedException { - restorePartitionStates(cacheGroups(), logicalState.partitionRecoveryStates()); + @Override public void afterLogicalUpdatesApplied( + IgniteCacheDatabaseSharedManager mgr, + GridCacheDatabaseSharedManager.RestoreLogicalState restoreState) throws IgniteCheckedException { + restorePartitionStates(cacheGroups(), restoreState.partitionRecoveryStates()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java index b3e3e108beeef..9ef0923ba4712 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java @@ -80,6 +80,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.txlog.TxState; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataRow; @@ -349,6 +350,20 @@ public MvccProcessorImpl(GridKernalContext ctx) { txLogPageStoreInit(mgr); } + /** {@inheritDoc} */ + @Override public void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr, + GridCacheDatabaseSharedManager.RestoreBinaryState restoreState) throws IgniteCheckedException { + + boolean hasMvccCaches = ctx.cache().persistentCaches().stream() + .anyMatch(c -> c.cacheConfiguration().getAtomicityMode() == TRANSACTIONAL_SNAPSHOT); + + if (hasMvccCaches) { + txLog = new TxLog(ctx, mgr); + + mvccEnabled = true; + } + } + /** * @param mgr Database shared manager. * @throws IgniteCheckedException If failed. @@ -358,15 +373,6 @@ private void txLogPageStoreInit(IgniteCacheDatabaseSharedManager mgr) throws Ign ctx.cache().context().pageStore().initialize(TX_LOG_CACHE_ID, 1, TX_LOG_CACHE_NAME, mgr.dataRegion(TX_LOG_CACHE_NAME).memoryMetrics()); - - boolean hasMvccCaches = ctx.cache().cacheGroups().stream().filter(CacheGroupContext::persistenceEnabled) - .anyMatch(g -> g.config().getAtomicityMode() == TRANSACTIONAL_SNAPSHOT); - - if (hasMvccCaches) { - txLog = new TxLog(ctx, mgr); - - mvccEnabled = true; - } } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java index 676210964e051..269dbd318f520 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java @@ -51,20 +51,24 @@ public default void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager m /** * Callback executed when binary memory has fully restored and WAL logging is resumed. * - * @param binaryState Result of binary recovery. + * + * @param mgr Database shared manager. + * @param restoreState Result of binary recovery. * @throws IgniteCheckedException If failed. */ - public default void afterBinaryMemoryRestore(GridCacheDatabaseSharedManager.RestoreBinaryState binaryState) - throws IgniteCheckedException {} + public default void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr, + GridCacheDatabaseSharedManager.RestoreBinaryState restoreState) throws IgniteCheckedException {} /** * Callback executed when all logical updates were applied and page memory become to fully consistent state. * - * @param logicalState Result of logical recovery. + * + * @param mgr Database shared manager. + * @param restoreState Result of logical recovery. * @throws IgniteCheckedException If failed. */ - public default void afterLogicalUpdatesApplied(GridCacheDatabaseSharedManager.RestoreLogicalState logicalState) - throws IgniteCheckedException {} + public default void afterLogicalUpdatesApplied(IgniteCacheDatabaseSharedManager mgr, + GridCacheDatabaseSharedManager.RestoreLogicalState restoreState) throws IgniteCheckedException {} /** * Callback executed when all physical updates are applied and we are ready to write new physical records diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 5d82bee7a9ec4..722c82cdc4aed 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -936,7 +936,7 @@ else if (restored != null) cctx.wal().resumeLogging(restored); for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) - lsnr.afterBinaryMemoryRestore(binaryState); + lsnr.afterBinaryMemoryRestore(this, binaryState); if (log.isInfoEnabled()) log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms."); @@ -2419,7 +2419,7 @@ private RestoreLogicalState applyLogicalUpdates( ", time=" + (U.currentTimeMillis() - start) + " ms]"); for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) - lsnr.afterLogicalUpdatesApplied(restoreLogicalState); + lsnr.afterLogicalUpdatesApplied(this, restoreLogicalState); return restoreLogicalState; } @@ -4660,7 +4660,9 @@ private class MetastorageRecoveryLifecycle implements DatabaseLifecycleListener cctx.pageStore().initializeForMetastorage(); } - @Override public void afterBinaryMemoryRestore(RestoreBinaryState binaryState) throws IgniteCheckedException { + @Override public void afterBinaryMemoryRestore( + IgniteCacheDatabaseSharedManager mgr, + RestoreBinaryState restoreState) throws IgniteCheckedException { assert metaStorage == null; metaStorage = createMetastorage(false); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index 07947a94b10ee..183c2eb6e82ff 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -42,7 +42,7 @@ public enum Feature { CACHE_EVENTS, EVICTION, EXPIRATION, - METRICS + METRICS, INTERCEPTOR } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java index d8ba0601287e7..9afb749e8cd5f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java @@ -115,7 +115,7 @@ public void testIndexRebuild() throws Exception { IgniteEx srv = startServer(); execute(srv, "CREATE TABLE T(k int primary key, v int) WITH \"cache_name=T,wrap_value=false," + - "atomicity=transactional_snapshot\""); + "atomicity=transactional\""); execute(srv, "CREATE INDEX IDX ON T(v)"); From 75c113656c2960aad49d2011146c0b8c06ef51d5 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Tue, 4 Dec 2018 22:33:45 +0700 Subject: [PATCH 203/403] IGNITE-10525 Web Console: "Import models" dialog should be a singleton. (cherry picked from commit ea872cc51c6fef019ac368bfe17e9db035c6256e) --- .../components/modal-import-models/service.js | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js index e17721dbc2b33..c45abaa9b761f 100644 --- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js +++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/service.js @@ -16,15 +16,23 @@ */ export default class ModalImportModels { - static $inject = ['$modal', '$uiRouter', 'AgentManager']; + static $inject = ['$modal', '$q', '$uiRouter', 'AgentManager']; - constructor($modal, $uiRouter, AgentManager) { + deferred; + + constructor($modal, $q, $uiRouter, AgentManager) { this.$modal = $modal; + this.$q = $q; this.$uiRouter = $uiRouter; this.AgentManager = AgentManager; } _goToDynamicState() { + if (this.deferred) + return this.deferred.promise; + + this.deferred = this.$q.defer(); + if (this._state) this.$uiRouter.stateRegistry.deregister(this._state); @@ -36,7 +44,6 @@ export default class ModalImportModels { }, onExit: () => { this.AgentManager.stopWatch(); - this._modal && this._modal.hide(); } }); @@ -45,21 +52,34 @@ export default class ModalImportModels { } _open() { + const self = this; + this._modal = this.$modal({ template: ` `, - controller: ['$state', function($state) {this.$state = $state;}], + controller: ['$state', function($state) { + this.$state = $state; + + this.onHide = () => { + self.deferred.resolve(true); + + this.$state.go('^'); + }; + }], controllerAs: '$ctrl', + backdrop: 'static', show: false }); return this.AgentManager.startAgentWatch('Back', this.$uiRouter.globals.current.name) .then(() => this._modal.$promise) - .then(() => this._modal.show()); + .then(() => this._modal.show()) + .then(() => this.deferred.promise) + .finally(() => this.deferred = null); } open() { From fa79d4e95994fbfce7148ca479c1f0e5ac9bbbcc Mon Sep 17 00:00:00 2001 From: Andrey Kuznetsov Date: Tue, 4 Dec 2018 19:46:45 +0300 Subject: [PATCH 204/403] IGNITE-10386 Add mode when WAL won't be disabled during rebalancing caused by BLT change Signed-off-by: Ivan Rakov (cherry picked from commit 7f758bf) Signed-off-by: Ivan Rakov --- .../apache/ignite/IgniteSystemProperties.java | 7 ++ .../processors/cache/WalStateManager.java | 7 +- .../GridDhtPartitionsExchangeFuture.java | 2 +- ...alModeChangeDuringRebalancingSelfTest.java | 102 +++++++++++++++++- 4 files changed, 114 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index c5df7b5a07004..9384ee749ad10 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -259,6 +259,13 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_TX_DEADLOCK_DETECTION_TIMEOUT = "IGNITE_TX_DEADLOCK_DETECTION_TIMEOUT"; + /** + * System property to enable pending transaction tracker. + * Affects impact of {@link IgniteSystemProperties#IGNITE_DISABLE_WAL_DURING_REBALANCING} property: + * if this property is set, WAL anyway won't be disabled during rebalancing triggered by baseline topology change. + */ + public static final String IGNITE_PENDING_TX_TRACKER_ENABLED = "IGNITE_PENDING_TX_TRACKER_ENABLED"; + /** * System property to override multicast group taken from configuration. * Used for testing purposes. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java index d17f20f221666..4b39f39b484c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java @@ -387,9 +387,12 @@ else if (!F.eq(grpDesc.deploymentId(), curGrpDesc.deploymentId())) { * in OWNING state if such feature is enabled. * * @param topVer Topology version. + * @param changedBaseline The exchange is caused by Baseline Topology change. */ - public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) { - if (!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, false)) + public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer, boolean changedBaseline) { + if (changedBaseline + && IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_PENDING_TX_TRACKER_ENABLED) + || !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, false)) return; Set grpsToEnableWal = new HashSet<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 31ce07e9d5ef7..6c408aa4482e4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2105,7 +2105,7 @@ public void finishMerged() { grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false); } - cctx.walState().changeLocalStatesOnExchangeDone(res); + cctx.walState().changeLocalStatesOnExchangeDone(res, changedBaseline()); } final Throwable err0 = err; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java index 48b60d48c9981..0a1d5641a85a3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java @@ -23,6 +23,7 @@ import java.nio.MappedByteBuffer; import java.nio.file.OpenOption; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -51,6 +52,7 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Assert; @@ -63,6 +65,12 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr /** */ private static boolean disableWalDuringRebalancing = true; + /** */ + private static boolean enablePendingTxTracker = false; + + /** */ + private static int dfltCacheBackupCnt = 0; + /** */ private static final AtomicReference supplyMessageLatch = new AtomicReference<>(); @@ -92,7 +100,8 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr cfg.setCacheConfiguration( new CacheConfiguration(DEFAULT_CACHE_NAME) // Test checks internal state before and after rebalance, so it is configured to be triggered manually - .setRebalanceDelay(-1), + .setRebalanceDelay(-1) + .setBackups(dfltCacheBackupCnt), new CacheConfiguration(REPL_CACHE) .setRebalanceDelay(-1) @@ -147,6 +156,9 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr System.setProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, Boolean.toString(disableWalDuringRebalancing)); + System.setProperty(IgniteSystemProperties.IGNITE_PENDING_TX_TRACKER_ENABLED, + Boolean.toString(enablePendingTxTracker)); + return cfg; } @@ -184,6 +196,17 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr cleanPersistenceDir(); disableWalDuringRebalancing = true; + enablePendingTxTracker = false; + dfltCacheBackupCnt = 0; + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + System.clearProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING); + + System.clearProperty(IgniteSystemProperties.IGNITE_PENDING_TX_TRACKER_ENABLED); } /** @@ -283,6 +306,60 @@ else if (timestamp >= rebalanceStartedTimestamp && timestamp <= rebalanceFinishe assertEquals(disableWalDuringRebalancing ? 1 : 0, checkpointsAfterRebalance); // checkpoint if WAL was re-activated } + /** + * @throws Exception If failed. + */ + public void testWalDisabledDuringRebalancingWithPendingTxTracker() throws Exception { + enablePendingTxTracker = true; + dfltCacheBackupCnt = 2; + + Ignite ignite = startGrids(3); + + ignite.cluster().active(true); + + ignite.cluster().setBaselineTopology(3); + + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); + + stopGrid(2); + + awaitExchange((IgniteEx)ignite); + + doLoad(cache, 4, 10_000); + + IgniteEx newIgnite = startGrid(2); + + awaitExchange(newIgnite); + + CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group(); + + assertFalse(grpCtx.walEnabled()); + + long rebalanceStartedTs = System.currentTimeMillis(); + + for (Ignite g : G.allGrids()) + g.cache(DEFAULT_CACHE_NAME).rebalance(); + + awaitPartitionMapExchange(); + + assertTrue(grpCtx.walEnabled()); + + long rebalanceFinishedTs = System.currentTimeMillis(); + + CheckpointHistory cpHist = + ((GridCacheDatabaseSharedManager)newIgnite.context().cache().context().database()).checkpointHistory(); + + assertNotNull(cpHist); + + // Ensure there was a checkpoint on WAL re-activation. + assertEquals( + 1, + cpHist.checkpoints() + .stream() + .filter(ts -> rebalanceStartedTs <= ts && ts <= rebalanceFinishedTs) + .count()); + } + /** * @throws Exception If failed. */ @@ -531,6 +608,29 @@ private void awaitExchange(IgniteEx ig) throws IgniteCheckedException { ig.context().cache().context().exchange().lastTopologyFuture().get(); } + /** + * Put random values to cache in multiple threads until time interval given expires. + * + * @param cache Cache to modify. + * @param threadCnt Number ot threads to be used. + * @param duration Time interval in milliseconds. + * @throws Exception When something goes wrong. + */ + private void doLoad(IgniteCache cache, int threadCnt, long duration) throws Exception { + GridTestUtils.runMultiThreaded(() -> { + long stopTs = U.currentTimeMillis() + duration; + + int keysCnt = getKeysCount(); + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + do { + cache.put(rnd.nextInt(keysCnt), rnd.nextInt()); + } + while (U.currentTimeMillis() < stopTs); + }, threadCnt, "load-cache"); + } + /** * */ From 9ef62f7cbaeece4d14e40eada88c75cac426cbe2 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Wed, 5 Dec 2018 16:29:58 +0700 Subject: [PATCH 205/403] IGNITE-10536 Web Console: Correct handle error in agent manager service. (cherry picked from commit 40f47b466552270ae74d5f8b860e04d0b46e367e) --- .../frontend/app/modules/agent/AgentManager.service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index dad625d8076f7..ef0d4237609fa 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -552,7 +552,7 @@ export default class AgentManager { if (err instanceof CancellationError) return; - return err; + throw err; }); } From 3b3b4094fa4c695a707adbf9cce59fcf8e41fdec Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Fri, 30 Nov 2018 15:46:21 +0300 Subject: [PATCH 206/403] IGNITE-10484 Fixed activate/deactivate hang - Fixes #5534. Signed-off-by: Alexey Goncharuk --- .../org/apache/ignite/internal/IgniteKernal.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 59b16abd32b77..8c6629f31813a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1090,6 +1090,14 @@ public void start( IgniteInternalFuture transitionWaitFut = joinData.transitionWaitFuture(); + // Notify discovery manager the first to make sure that topology is discovered. + // Active flag is not used in managers, so it is safe to pass true. + ctx.discovery().onKernalStart(true); + + // Notify IO manager the second so further components can send and receive messages. + // Must notify the IO manager before transition state await to make sure IO connection can be established. + ctx.io().onKernalStart(true); + boolean active; if (transitionWaitFut != null) { @@ -1103,12 +1111,6 @@ public void start( else active = joinData.active(); - // Notify discovery manager the first to make sure that topology is discovered. - ctx.discovery().onKernalStart(active); - - // Notify IO manager the second so further components can send and receive messages. - ctx.io().onKernalStart(active); - boolean recon = false; // Callbacks. From 5c27c1b85ac69bd297f17543a680007f180085db Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Thu, 6 Dec 2018 11:30:08 +0700 Subject: [PATCH 207/403] IGNITE-10564 Added tasks to collect rebalance metrics. Co-authored-by: Alexey Kuznetsov (cherry picked from commit ececcbcd1883a2cda5a2eb12e0f0deffddf6b829) --- ...VisorCacheRebalanceCollectorJobResult.java | 91 ++++++++ .../VisorCacheRebalanceCollectorTask.java | 194 ++++++++++++++++++ .../VisorCacheRebalanceCollectorTaskArg.java | 54 +++++ ...isorCacheRebalanceCollectorTaskResult.java | 92 +++++++++ .../visor/node/VisorNodeBaselineStatus.java | 45 ++++ .../visor/node/VisorNodeDataCollectorJob.java | 74 ++++--- .../VisorNodeDataCollectorTaskResult.java | 3 +- .../internal/visor/util/VisorTaskUtils.java | 43 ++++ 8 files changed, 564 insertions(+), 32 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java new file mode 100644 index 0000000000000..5bd818d424848 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.node; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Result object for cache rebalance job. + */ +public class VisorCacheRebalanceCollectorJobResult extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** Rebalance percent. */ + private double rebalance; + + /** Node baseline state. */ + private VisorNodeBaselineStatus baseline; + + /** + * Default constructor. + */ + public VisorCacheRebalanceCollectorJobResult() { + // No-op. + } + + /** + * @return Rebalance progress. + */ + public double getRebalance() { + return rebalance; + } + + /** + * @param rebalance Rebalance progress. + */ + public void setRebalance(double rebalance) { + this.rebalance = rebalance; + } + + /** + * @return Node baseline status. + */ + public VisorNodeBaselineStatus getBaseline() { + return baseline; + } + + /** + * @param baseline Node baseline status. + */ + public void setBaseline(VisorNodeBaselineStatus baseline) { + this.baseline = baseline; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + out.writeDouble(rebalance); + U.writeEnum(out, baseline); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + rebalance = in.readDouble(); + baseline = VisorNodeBaselineStatus.fromOrdinal(in.readByte()); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorCacheRebalanceCollectorJobResult.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java new file mode 100644 index 0000000000000..eda9f9419c100 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.node; + +import java.util.Collection; +import java.util.List; +import org.apache.ignite.cache.CacheMetrics; +import org.apache.ignite.cluster.BaselineNode; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.cluster.IgniteClusterEx; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheAdapter; +import org.apache.ignite.internal.processors.cache.GridCacheProcessor; +import org.apache.ignite.internal.processors.cache.GridCacheUtils; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorMultiNodeTask; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.visor.node.VisorNodeBaselineStatus.BASELINE_NOT_AVAILABLE; +import static org.apache.ignite.internal.visor.node.VisorNodeBaselineStatus.NODE_IN_BASELINE; +import static org.apache.ignite.internal.visor.node.VisorNodeBaselineStatus.NODE_NOT_IN_BASELINE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.MINIMAL_REBALANCE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.NOTHING_TO_REBALANCE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_COMPLETE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_NOT_AVAILABLE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isProxyCache; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isRestartingCache; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.log; + +/** + * Collects topology rebalance metrics. + */ +@GridInternal +public class VisorCacheRebalanceCollectorTask extends VisorMultiNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorCacheRebalanceCollectorJob job(VisorCacheRebalanceCollectorTaskArg arg) { + return new VisorCacheRebalanceCollectorJob(arg, debug); + } + + /** {@inheritDoc} */ + @Nullable @Override protected VisorCacheRebalanceCollectorTaskResult reduce0(List results) { + return reduce(new VisorCacheRebalanceCollectorTaskResult(), results); + } + + /** + * @param taskRes Task result. + * @param results Results. + * @return Topology rebalance metrics collector task result. + */ + protected VisorCacheRebalanceCollectorTaskResult reduce( + VisorCacheRebalanceCollectorTaskResult taskRes, + List results + ) { + for (ComputeJobResult res : results) { + VisorCacheRebalanceCollectorJobResult jobRes = res.getData(); + + if (jobRes != null) { + if (res.getException() == null) + taskRes.getRebalance().put(res.getNode().id(), jobRes.getRebalance()); + + taskRes.getBaseline().put(res.getNode().id(), jobRes.getBaseline()); + } + } + + return taskRes; + } + + /** + * Job that collects rebalance metrics. + */ + private static class VisorCacheRebalanceCollectorJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** + * Create job with given argument. + * + * @param arg Job argument. + * @param debug Debug flag. + */ + private VisorCacheRebalanceCollectorJob(VisorCacheRebalanceCollectorTaskArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected VisorCacheRebalanceCollectorJobResult run(VisorCacheRebalanceCollectorTaskArg arg) { + VisorCacheRebalanceCollectorJobResult res = new VisorCacheRebalanceCollectorJobResult(); + + long start0 = U.currentTimeMillis(); + + try { + int partitions = 0; + double total = 0; + double ready = 0; + + GridCacheProcessor cacheProc = ignite.context().cache(); + + boolean rebalanceInProgress = false; + + for (CacheGroupContext grp: cacheProc.cacheGroups()) { + String cacheName = grp.config().getName(); + + if (isProxyCache(ignite, cacheName) || isRestartingCache(ignite, cacheName)) + continue; + + try { + GridCacheAdapter ca = cacheProc.internalCache(cacheName); + + if (ca == null || !ca.context().started()) + continue; + + CacheMetrics cm = ca.localMetrics(); + + partitions += cm.getTotalPartitionsCount(); + + long keysTotal = cm.getEstimatedRebalancingKeys(); + long keysReady = cm.getRebalancedKeys(); + + if (keysReady >= keysTotal) + keysReady = Math.max(keysTotal - 1, 0); + + total += keysTotal; + ready += keysReady; + + if (cm.getRebalancingPartitionsCount() > 0) + rebalanceInProgress = true; + } + catch(IllegalStateException | IllegalArgumentException e) { + if (debug && ignite.log() != null) + ignite.log().error("Ignored cache group: " + grp.cacheOrGroupName(), e); + } + } + + if (partitions == 0) + res.setRebalance(NOTHING_TO_REBALANCE); + else if (total == 0 && rebalanceInProgress) + res.setRebalance(MINIMAL_REBALANCE); + else + res.setRebalance(total > 0 ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE); + } + catch (Exception e) { + res.setRebalance(REBALANCE_NOT_AVAILABLE); + + ignite.log().error("Failed to collect rebalance metrics", e); + } + + if (GridCacheUtils.isPersistenceEnabled(ignite.configuration())) { + IgniteClusterEx cluster = ignite.cluster(); + + Object consistentId = ignite.localNode().consistentId(); + + Collection baseline = cluster.currentBaselineTopology(); + + boolean inBaseline = baseline.stream().anyMatch(n -> consistentId.equals(n.consistentId())); + + res.setBaseline(inBaseline ? NODE_IN_BASELINE : NODE_NOT_IN_BASELINE); + } + else + res.setBaseline(BASELINE_NOT_AVAILABLE); + + if (debug) + log(ignite.log(), "Collected rebalance metrics", getClass(), start0); + + return res; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorCacheRebalanceCollectorJob.class, this); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java new file mode 100644 index 0000000000000..d97fd50192572 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.node; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * Argument for {@link VisorCacheRebalanceCollectorTask} task. + */ +public class VisorCacheRebalanceCollectorTaskArg extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** + * Default constructor. + */ + public VisorCacheRebalanceCollectorTaskArg() { + // No-op. + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + // No-op. + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + // No-op. + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorCacheRebalanceCollectorTaskArg.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java new file mode 100644 index 0000000000000..1305cd2df87eb --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.node; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Result object for {@link VisorCacheRebalanceCollectorTask} task. + */ +public class VisorCacheRebalanceCollectorTaskResult extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** Rebalance state on nodes. */ + private Map rebalance = new HashMap<>(); + + /** Nodes baseline status. */ + private Map baseline = new HashMap<>(); + + /** + * Default constructor. + */ + public VisorCacheRebalanceCollectorTaskResult() { + // No-op. + } + + /** + * @return Rebalance on nodes. + */ + public Map getRebalance() { + return rebalance; + } + + /** + * @return Baseline. + */ + public Map getBaseline() { + return baseline; + } + + /** + * Add specified results. + * + * @param res Results to add. + */ + public void add(VisorCacheRebalanceCollectorTaskResult res) { + assert res != null; + + rebalance.putAll(res.getRebalance()); + baseline.putAll(res.getBaseline()); + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeMap(out, rebalance); + U.writeMap(out, baseline); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + rebalance = U.readMap(in); + baseline = U.readMap(in); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorCacheRebalanceCollectorTaskResult.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java new file mode 100644 index 0000000000000..ea90be3a6ef2c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.node; + +import org.jetbrains.annotations.Nullable; + +/** + * Node baseline status. + */ +public enum VisorNodeBaselineStatus { + /** */ + NODE_IN_BASELINE, + /** */ + NODE_NOT_IN_BASELINE, + /** */ + BASELINE_NOT_AVAILABLE; + + /** Enumerated values. */ + private static final VisorNodeBaselineStatus[] VALS = values(); + + /** + * Efficiently gets enumerated value from its ordinal. + * + * @param ord Ordinal value. + * @return Enumerated value or {@code null} if ordinal out of range. + */ + @Nullable public static VisorNodeBaselineStatus fromOrdinal(int ord) { + return ord >= 0 && ord < VALS.length ? VALS[ord] : null; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index 9025ed084b9c5..9a7d2b146618e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -28,8 +28,9 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; +import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter; @@ -50,9 +51,15 @@ import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isSystemCache; import static org.apache.ignite.internal.visor.compute.VisorComputeMonitoringHolder.COMPUTE_MONITORING_HOLDER_KEY; import static org.apache.ignite.internal.visor.util.VisorTaskUtils.EVT_MAPPER; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.MINIMAL_REBALANCE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.NOTHING_TO_REBALANCE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_COMPLETE; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_NOT_AVAILABLE; import static org.apache.ignite.internal.visor.util.VisorTaskUtils.VISOR_TASK_EVTS; import static org.apache.ignite.internal.visor.util.VisorTaskUtils.checkExplicitTaskMonitoring; import static org.apache.ignite.internal.visor.util.VisorTaskUtils.collectEvents; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isProxyCache; +import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isRestartingCache; import static org.apache.ignite.internal.visor.util.VisorTaskUtils.log; /** @@ -140,18 +147,6 @@ protected boolean compatibleWith(IgniteProductVersion ver) { return false; } - /** - * @param cacheName Cache name to check. - * @return {@code true} if cache on local node is not a data cache or near cache disabled. - */ - private boolean proxyCache(String cacheName) { - GridDiscoveryManager discovery = ignite.context().discovery(); - - ClusterNode locNode = ignite.localNode(); - - return !(discovery.cacheAffinityNode(locNode, cacheName) || discovery.cacheNearNode(locNode, cacheName)); - } - /** * Collect memory metrics. * @@ -194,38 +189,51 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto List resCaches = res.getCaches(); - for (String cacheName : cacheProc.cacheNames()) { - if (proxyCache(cacheName)) - continue; + boolean rebalanceInProgress = false; - boolean sysCache = isSystemCache(cacheName); + for (CacheGroupContext grp : cacheProc.cacheGroups()) { + boolean first = true; - if (arg.getSystemCaches() || !(sysCache || isIgfsCache(cfg, cacheName))) { + for (GridCacheContext cache : grp.caches()) { long start0 = U.currentTimeMillis(); + String cacheName = cache.name(); + try { + if (isProxyCache(ignite, cacheName) || isRestartingCache(ignite, cacheName)) + continue; + GridCacheAdapter ca = cacheProc.internalCache(cacheName); if (ca == null || !ca.context().started()) continue; - CacheMetrics cm = ca.localMetrics(); + if (first) { + CacheMetrics cm = ca.localMetrics(); + + partitions += cm.getTotalPartitionsCount(); + + long keysTotal = cm.getEstimatedRebalancingKeys(); + long keysReady = cm.getRebalancedKeys(); + + if (keysReady >= keysTotal) + keysReady = Math.max(keysTotal - 1, 0); - partitions += cm.getTotalPartitionsCount(); + total += keysTotal; + ready += keysReady; - long partTotal = cm.getEstimatedRebalancingKeys(); - long partReady = cm.getRebalancedKeys(); + if (!rebalanceInProgress && cm.getRebalancingPartitionsCount() > 0) + rebalanceInProgress = true; - if (partReady >= partTotal) - partReady = Math.max(partTotal - 1, 0); + first = false; + } - total += partTotal; - ready += partReady; + boolean addToRes = arg.getSystemCaches() || !(isSystemCache(cacheName) || isIgfsCache(cfg, cacheName)); - if (all || cacheGrps.contains(ca.configuration().getGroupName())) + if (addToRes && (all || cacheGrps.contains(ca.configuration().getGroupName()))) resCaches.add(new VisorCache(ignite, ca, arg.isCollectCacheMetrics())); } - catch(IllegalStateException | IllegalArgumentException e) { + catch (IllegalStateException | IllegalArgumentException e) { if (debug && ignite.log() != null) ignite.log().error("Ignored cache: " + cacheName, e); } @@ -237,11 +245,14 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto } if (partitions == 0) - res.setRebalance(-1); + res.setRebalance(NOTHING_TO_REBALANCE); + else if (total == 0 && rebalanceInProgress) + res.setRebalance(MINIMAL_REBALANCE); else - res.setRebalance(total > 0 ? ready / total : 1); + res.setRebalance(total > 0 ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE); } catch (Exception e) { + res.setRebalance(REBALANCE_NOT_AVAILABLE); res.setCachesEx(new VisorExceptionWrapper(e)); } } @@ -260,7 +271,8 @@ protected void igfs(VisorNodeDataCollectorJobResult res) { FileSystemConfiguration igfsCfg = igfs.configuration(); - if (proxyCache(igfsCfg.getDataCacheConfiguration().getName()) || proxyCache(igfsCfg.getMetaCacheConfiguration().getName())) + if (isProxyCache(ignite, igfsCfg.getDataCacheConfiguration().getName()) || + isProxyCache(ignite, igfsCfg.getMetaCacheConfiguration().getName())) continue; try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java index eb161f82c6638..f8eb8690ce2ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java @@ -131,7 +131,8 @@ public boolean isEmpty() { readyTopVers.isEmpty() && pendingExchanges.isEmpty() && persistenceMetrics.isEmpty() && - persistenceMetricsEx.isEmpty(); + persistenceMetricsEx.isEmpty() && + rebalance.isEmpty(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java index fda9ba199bc0f..7ab1ffcb18b84 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java @@ -58,6 +58,10 @@ import org.apache.ignite.cache.eviction.AbstractEvictionPolicyFactory; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.Event; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl; import org.apache.ignite.internal.processors.igfs.IgfsEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; @@ -109,6 +113,19 @@ public class VisorTaskUtils { /** Log files count limit */ public static final int LOG_FILES_COUNT_LIMIT = 5000; + /** */ + public static final int NOTHING_TO_REBALANCE = -1; + + /** */ + public static final int REBALANCE_NOT_AVAILABLE = -2; + + /** */ + public static final double MINIMAL_REBALANCE = 0.01; + + /** */ + public static final int REBALANCE_COMPLETE = 1; + + /** */ private static final int DFLT_BUFFER_SIZE = 4096; @@ -1248,4 +1265,30 @@ public static Collection splitAddresses(String s) { return Arrays.asList(addrs); } + + /** + * @param ignite Ignite. + * @param cacheName Cache name to check. + * @return {@code true} if cache on local node is not a data cache or near cache disabled. + */ + public static boolean isProxyCache(IgniteEx ignite, String cacheName) { + GridDiscoveryManager discovery = ignite.context().discovery(); + + ClusterNode locNode = ignite.localNode(); + + return !(discovery.cacheAffinityNode(locNode, cacheName) || discovery.cacheNearNode(locNode, cacheName)); + } + + /** + * Check whether cache restarting in progress. + * + * @param ignite Grid. + * @param cacheName Cache name to check. + * @return {@code true} when cache restarting in progress. + */ + public static boolean isRestartingCache(IgniteEx ignite, String cacheName) { + IgniteCacheProxy proxy = ignite.context().cache().jcache(cacheName); + + return proxy instanceof IgniteCacheProxyImpl && ((IgniteCacheProxyImpl) proxy).isRestarting(); + } } From 0abe70a6090818573e5fde818a39d430c6f9fe6f Mon Sep 17 00:00:00 2001 From: Pavel Konstantinov Date: Thu, 6 Dec 2018 11:34:53 +0700 Subject: [PATCH 208/403] IGNITE-10373 Web Console: Fixed docker-compose.yml. (cherry picked from commit 63b1422e9095bb9c9afa7cfc27e7f24b30ce8f97) --- modules/web-console/docker/compose/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/web-console/docker/compose/docker-compose.yml b/modules/web-console/docker/compose/docker-compose.yml index 109bfca85ab2d..15f2ae4de6436 100644 --- a/modules/web-console/docker/compose/docker-compose.yml +++ b/modules/web-console/docker/compose/docker-compose.yml @@ -15,7 +15,7 @@ # limitations under the License. # -version: 1 +version: '3' services: mongodb: @@ -50,7 +50,6 @@ services: image: apacheignite/web-console-frontend depends_on: - mongodb - - testenv ports: # Proxy HTTP nginx port (HOST_PORT:DOCKER_PORT) - 80:80 From 3410b8fa2734b3fa00581e62ef825807b35768e6 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Thu, 6 Dec 2018 11:41:48 +0700 Subject: [PATCH 209/403] IGNITE-8370 Web Console: Updated type. (cherry picked from commit 24a50e1045a759aff584b5d651cc46ce9047394b) --- modules/web-console/frontend/app/types/index.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/web-console/frontend/app/types/index.ts b/modules/web-console/frontend/app/types/index.ts index 333c00be875d2..df6a169d6a215 100644 --- a/modules/web-console/frontend/app/types/index.ts +++ b/modules/web-console/frontend/app/types/index.ts @@ -23,11 +23,12 @@ interface ITfMetatagsConfig { export interface IIgniteNg1StateDeclaration extends Ng1StateDeclaration { /** - * Whether to store state as last visited in local storage or not. - * true - will be saved - * false (default) - won't be saved + * Whether to store state as last visited in local storage or not: + * `true` - will be saved + * `false` (default) - won't be saved * @type {boolean} */ unsaved?: boolean, - tfMetaTags: ITfMetatagsConfig + tfMetaTags: ITfMetatagsConfig, + permission?: string } From fff9c1114f5067725f2cfef65a77d43788375d92 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Tue, 27 Nov 2018 19:31:51 +0300 Subject: [PATCH 210/403] IGNITE-10390 Fixed BPlusTree#isEmpty - Fixes #5486. Signed-off-by: Alexey Goncharuk (cherry picked from commit 0b4282be26b4d042637ce57821e9a3448043d56d) --- .../persistence/GridCacheOffheapManager.java | 2 +- .../cache/persistence/tree/BPlusTree.java | 23 +++++++++++-------- .../database/BPlusTreeSelfTest.java | 23 +++++++++++++++++++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 2dfb41f3d106e..e1bd8dd77db35 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -1568,7 +1568,7 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException pendingTree = pendingTree0; - if (!hasPendingEntries && pendingTree0.size() > 0) + if (!hasPendingEntries && !pendingTree0.isEmpty()) hasPendingEntries = true; int grpId = grp.groupId(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java index ce9463dc1e78b..02dab9466c307 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java @@ -2109,21 +2109,26 @@ public final boolean isEmpty() throws IgniteCheckedException { long rootId, rootPage = acquirePage(rootId = treeMeta.rootId); - long rootAddr = readLock(rootId, rootPage); + try { + long rootAddr = readLock(rootId, rootPage); - if (rootAddr == 0) { - checkDestroyed(); + if (rootAddr == 0) { + checkDestroyed(); - continue; - } + continue; + } - try { - BPlusIO io = io(rootAddr); + try { + BPlusIO io = io(rootAddr); - return io.getCount(rootAddr) == 0; + return io.getCount(rootAddr) == 0; + } + finally { + readUnlock(rootId, rootPage, rootAddr); + } } finally { - readUnlock(rootId, rootPage, rootAddr); + releasePage(rootId, rootPage); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java index 6b694c9d92e13..3a2907012ecd5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java @@ -251,6 +251,29 @@ public void testRetries() throws IgniteCheckedException { } } + /** + * @throws Exception if failed. + */ + public void testIsEmpty() throws Exception { + TestTree tree = createTestTree(true); + + assertTrue(tree.isEmpty()); + + for (long i = 1; i <= 500; i++) { + tree.put(i); + + assertFalse(tree.isEmpty()); + } + + for (long i = 1; i <= 500; i++) { + assertFalse(tree.isEmpty()); + + tree.remove(i); + } + + assertTrue(tree.isEmpty()); + } + /** * @throws IgniteCheckedException If failed. */ From dd0b6d562c1be0c363e885e5aa3babb6893332c5 Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Sat, 1 Dec 2018 19:33:14 +0300 Subject: [PATCH 211/403] IGNITE-10374 Fixed race leading to rebalancing hang when WAL is disabled during rebalancing - Fixes #5468. (cherry-picked from commit #493e0aa) --- .../processors/cache/WalStateManager.java | 4 +- .../GridDhtPartitionsExchangeFuture.java | 3 +- .../dht/preloader/GridDhtPreloader.java | 35 +-- .../topology/GridClientPartitionTopology.java | 2 +- .../topology/GridDhtPartitionTopology.java | 4 +- .../GridDhtPartitionTopologyImpl.java | 17 +- .../cache/persistence/CheckpointFuture.java | 2 +- .../GridCacheDatabaseSharedManager.java | 2 +- ...ePdsCacheWalDisabledOnRebalancingTest.java | 266 ++++++++++++++++++ .../testsuites/IgnitePdsTestSuite4.java | 3 + 10 files changed, 293 insertions(+), 45 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java index 4b39f39b484c0..0bcd07da18fcd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java @@ -476,7 +476,7 @@ else if (!grp.localWalEnabled()) public void onGroupRebalanceFinished(int grpId, AffinityTopologyVersion topVer) { TemporaryDisabledWal session0 = tmpDisabledWal; - if (session0 == null || !session0.topVer.equals(topVer)) + if (session0 == null || session0.topVer.compareTo(topVer) > 0) return; session0.remainingGrps.remove(grpId); @@ -509,7 +509,7 @@ public void onGroupRebalanceFinished(int grpId, AffinityTopologyVersion topVer) assert grp != null; - grp.topology().ownMoving(session0.topVer); + grp.topology().ownMoving(topVer); } cctx.exchange().refreshPartitions(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 6c408aa4482e4..eeaec4ec594db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2105,7 +2105,8 @@ public void finishMerged() { grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false); } - cctx.walState().changeLocalStatesOnExchangeDone(res, changedBaseline()); + if (changedAffinity()) + cctx.walState().changeLocalStatesOnExchangeDone(res, changedBaseline()); } final Throwable err0 = err; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java index fa0c2b7b55976..cb26c0dd30e88 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java @@ -23,7 +23,6 @@ import java.util.UUID; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; @@ -187,38 +186,10 @@ private IgniteCheckedException stopError() { if (rebFut.isDone() && !rebFut.result()) return true; // Required, previous rebalance cancelled. - final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion(); + AffinityTopologyVersion lastAffChangeTopVer = + ctx.exchange().lastAffinityChangedTopologyVersion(exchFut.topologyVersion()); - Collection aliveNodes = ctx.discovery().aliveServerNodes().stream() - .map(ClusterNode::id) - .collect(Collectors.toList()); - - return assignmentsChanged(rebTopVer, exchTopVer) || - !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete. - } - - /** - * @param oldTopVer Previous topology version. - * @param newTopVer New topology version to check result. - * @return {@code True} if affinity assignments changed between two versions for local node. - */ - private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) { - final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer); - - // We should get affinity assignments based on previous rebalance to calculate difference. - // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant. - final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ? - grp.affinity().cachedAffinity(oldTopVer) : null; - - if (prevAff == null) - return false; - - boolean assignsChanged = false; - - for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++) - assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode()); - - return assignsChanged; + return lastAffChangeTopVer.compareTo(rebTopVer) > 0; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java index 01908af8c9d4d..0a50b94e20b17 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java @@ -1112,7 +1112,7 @@ private void removeNode(UUID nodeId) { } /** {@inheritDoc} */ - @Override public void ownMoving(AffinityTopologyVersion topVer) { + @Override public void ownMoving(AffinityTopologyVersion rebFinishedTopVer) { // No-op } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index 81eab849a3561..4b1e5a6ada99d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -373,9 +373,9 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, /** * Owns all moving partitions for the given topology version. * - * @param topVer Topology version. + * @param rebFinishedTopVer Topology version when rebalancing finished. */ - public void ownMoving(AffinityTopologyVersion topVer); + public void ownMoving(AffinityTopologyVersion rebFinishedTopVer); /** * @param part Evicted partition. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index cfb5a961d1789..267e7618ff49b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -2575,19 +2575,26 @@ private void removeNode(UUID nodeId) { } /** {@inheritDoc} */ - @Override public void ownMoving(AffinityTopologyVersion topVer) { + @Override public void ownMoving(AffinityTopologyVersion rebFinishedTopVer) { lock.writeLock().lock(); + AffinityTopologyVersion lastAffChangeVer = ctx.exchange().lastAffinityChangedTopologyVersion(lastTopChangeVer); + + if (lastAffChangeVer.compareTo(rebFinishedTopVer) > 0) + log.info("Affinity topology changed, no MOVING partitions will be owned " + + "[rebFinishedTopVer=" + rebFinishedTopVer + + ", lastAffChangeVer=" + lastAffChangeVer + "]"); + try { for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) { if (locPart.state() == MOVING) { boolean reserved = locPart.reserve(); try { - if (reserved && locPart.state() == MOVING && lastTopChangeVer.equals(topVer)) - grp.topology().own(locPart); - else // topology changed, rebalancing must be restarted - return; + if (reserved && locPart.state() == MOVING && + lastAffChangeVer.compareTo(rebFinishedTopVer) <= 0 && + rebFinishedTopVer.compareTo(lastTopChangeVer) <= 0) + grp.topology().own(locPart); } finally { if (reserved) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java index 1c77013d63333..767173812ba8f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java @@ -31,5 +31,5 @@ public interface CheckpointFuture { /** * @return Finish future. */ - public GridFutureAdapter finishFuture(); + public GridFutureAdapter finishFuture(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 722c82cdc4aed..6d324b2747568 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -4228,7 +4228,7 @@ private static class CheckpointProgressSnapshot implements CheckpointFuture { } /** {@inheritDoc} */ - @Override public GridFutureAdapter finishFuture() { + @Override public GridFutureAdapter finishFuture() { return cpFinishFut; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java new file mode 100644 index 0000000000000..1246db1090cae --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheWalDisabledOnRebalancingTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.persistence.db; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.mxbean.CacheGroupMetricsMXBean; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; + +/** + * Test scenarios with rebalancing, IGNITE_DISABLE_WAL_DURING_REBALANCING optimization and topology changes + * such as client nodes join/leave, server nodes from BLT leave/join, server nodes out of BLT join/leave. + */ +public class IgnitePdsCacheWalDisabledOnRebalancingTest extends GridCommonAbstractTest { + /** Block message predicate to set to Communication SPI in node configuration. */ + private IgniteBiPredicate blockMessagePredicate; + + /** */ + private static final int CACHE1_PARTS_NUM = 8; + + /** */ + private static final int CACHE2_PARTS_NUM = 16; + + /** */ + private static final int CACHE3_PARTS_NUM = 32; + + /** */ + private static final int CACHE_SIZE = 2_000; + + /** */ + private static final String CACHE3_NAME = "cache3"; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + + System.setProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, "true"); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + System.clearProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration ccfg1 = new CacheConfiguration("cache1") + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setCacheMode(CacheMode.REPLICATED) + .setAffinity(new RendezvousAffinityFunction(false, CACHE1_PARTS_NUM)); + + CacheConfiguration ccfg2 = new CacheConfiguration("cache2") + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setCacheMode(CacheMode.PARTITIONED) + .setAffinity(new RendezvousAffinityFunction(false, CACHE2_PARTS_NUM)); + + CacheConfiguration ccfg3 = new CacheConfiguration(CACHE3_NAME) + .setBackups(2) + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setCacheMode(CacheMode.PARTITIONED) + .setAffinity(new RendezvousAffinityFunction(false, CACHE3_PARTS_NUM)); + + cfg.setCacheConfiguration(ccfg1, ccfg2, ccfg3); + + if ("client".equals(igniteInstanceName)) + cfg.setClientMode(true); + else { + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4) + .setWalMode(WALMode.LOG_ONLY) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(256 * 1024 * 1024)); + + cfg.setDataStorageConfiguration(dsCfg); + } + + TestRecordingCommunicationSpi commSpi = new TestRecordingCommunicationSpi(); + commSpi.blockMessages(blockMessagePredicate); + + cfg.setCommunicationSpi(commSpi); + + + return cfg; + } + + /** + * If client joins topology during rebalancing process, rebalancing finishes successfully, + * all partitions are owned as expected when rebalancing finishes. + */ + public void testClientJoinsLeavesDuringRebalancing() throws Exception { + Ignite ig0 = startGrids(2); + + ig0.active(true); + + for (int i = 0; i < 3; i++) + fillCache(ig0.getOrCreateCache("cache" + i), CACHE_SIZE); + + String ig1Name = "node01-" + grid(1).localNode().consistentId(); + + stopGrid(1); + + cleanPersistenceFiles(ig1Name); + + int groupId = ((IgniteEx) ig0).cachex(CACHE3_NAME).context().groupId(); + + blockMessagePredicate = (node, msg) -> { + if (msg instanceof GridDhtPartitionDemandMessage) + return ((GridDhtPartitionDemandMessage) msg).groupId() == groupId; + + return false; + }; + + IgniteEx ig1 = startGrid(1); + + startGrid("client"); + + stopGrid("client"); + + CacheGroupMetricsMXBean mxBean = ig1.cachex(CACHE3_NAME).context().group().mxBean(); + + assertTrue("Unexpected moving partitions count: " + mxBean.getLocalNodeMovingPartitionsCount(), + mxBean.getLocalNodeMovingPartitionsCount() == CACHE3_PARTS_NUM); + + TestRecordingCommunicationSpi commSpi = (TestRecordingCommunicationSpi) ig1 + .configuration().getCommunicationSpi(); + + commSpi.stopBlock(); + + boolean waitResult = GridTestUtils.waitForCondition( + () -> mxBean.getLocalNodeMovingPartitionsCount() == 0, + 30_000); + + assertTrue("Failed to wait for owning all partitions, parts in moving state: " + + mxBean.getLocalNodeMovingPartitionsCount(), waitResult); + } + + /** + * If server nodes from BLT leave topology and then join again after additional keys were put to caches, + * rebalance starts. + * + * Test verifies that all moving partitions get owned after rebalance finishes. + * + * @throws Exception If failed. + */ + public void testServerNodesFromBltLeavesAndJoinsDuringRebalancing() throws Exception { + Ignite ig0 = startGridsMultiThreaded(4); + + fillCache(ig0.cache(CACHE3_NAME), CACHE_SIZE); + + List nonAffinityKeys1 = nearKeys(grid(1).cache(CACHE3_NAME), 100, CACHE_SIZE / 2); + List nonAffinityKeys2 = nearKeys(grid(2).cache(CACHE3_NAME), 100, CACHE_SIZE / 2); + + stopGrid(1); + stopGrid(2); + + Set nonAffinityKeysSet = new HashSet<>(); + + nonAffinityKeysSet.addAll(nonAffinityKeys1); + nonAffinityKeysSet.addAll(nonAffinityKeys2); + + fillCache(ig0.cache(CACHE3_NAME), nonAffinityKeysSet); + + int groupId = ((IgniteEx) ig0).cachex(CACHE3_NAME).context().groupId(); + + blockMessagePredicate = (node, msg) -> { + if (msg instanceof GridDhtPartitionDemandMessage) + return ((GridDhtPartitionDemandMessage) msg).groupId() == groupId; + + return false; + }; + + IgniteEx ig1 = startGrid(1); + + CacheGroupMetricsMXBean mxBean = ig1.cachex(CACHE3_NAME).context().group().mxBean(); + + TestRecordingCommunicationSpi commSpi = (TestRecordingCommunicationSpi) ig1 + .configuration().getCommunicationSpi(); + + startGrid(2); + + commSpi.stopBlock(); + + boolean allOwned = GridTestUtils.waitForCondition( + () -> mxBean.getLocalNodeMovingPartitionsCount() == 0, 30_000); + + assertTrue("Partitions were not owned, there are " + mxBean.getLocalNodeMovingPartitionsCount() + + " partitions in MOVING state", allOwned); + } + + /** */ + private void cleanPersistenceFiles(String igName) throws Exception { + String ig1DbPath = Paths.get(DFLT_STORE_DIR, igName).toString(); + + File igDbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), ig1DbPath, false); + + U.delete(igDbDir); + Files.createDirectory(igDbDir.toPath()); + + String ig1DbWalPath = Paths.get(DFLT_STORE_DIR, "wal", igName).toString(); + + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), ig1DbWalPath, false)); + } + + /** */ + private void fillCache(IgniteCache cache, int cacheSize) { + for (int i = 0; i < cacheSize; i++) + cache.put(i, "value_" + i); + } + + /** */ + private void fillCache(IgniteCache cache, Collection keys) { + for (Integer key : keys) + cache.put(key, "value_" + key); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java index 027f34105b7fd..ad6bbbf0ec2b6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheWalDisabledOnRebalancingTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPartitionPreloadTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest; @@ -50,6 +51,8 @@ public static TestSuite suite() { suite.addTestSuite(ResetLostPartitionTest.class); + suite.addTestSuite(IgnitePdsCacheWalDisabledOnRebalancingTest.class); + return suite; } From f44bad04544ed49bd8e72d193edd2faea889ab8c Mon Sep 17 00:00:00 2001 From: Dmitriy Sorokin Date: Thu, 6 Dec 2018 17:43:27 +0300 Subject: [PATCH 212/403] IGNITE-10355 Tx rollback failure on put operations with caches whose topology fails validation - Fixes #5453. Signed-off-by: Ivan Rakov (cherry picked from commit bf75ef2) --- .../processors/cache/GridCacheUtils.java | 2 +- .../distributed/near/GridNearTxLocal.java | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java index ebf42fa16727a..596b619cb1254 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java @@ -1317,7 +1317,7 @@ public static long expireTimeInPast() { if (e instanceof CachePartialUpdateCheckedException) return new CachePartialUpdateException((CachePartialUpdateCheckedException)e); - else if (e instanceof ClusterTopologyServerNotFoundException) + else if (e.hasCause(ClusterTopologyServerNotFoundException.class)) return new CacheServerNotFoundException(e.getMessage(), e); else if (e instanceof SchemaOperationException) return new CacheException(e.getMessage(), e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java index b698ba13c109e..09c560181862f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java @@ -3919,16 +3919,23 @@ private IgniteInternalFuture chainFinishFuture(final NearTxFin fut.listen(new IgniteInClosure>() { @Override public void apply(IgniteInternalFuture fut0) { if (FINISH_FUT_UPD.compareAndSet(tx, fut, rollbackFut)) { - if (tx.state() == COMMITTED) { - if (log.isDebugEnabled()) - log.debug("Failed to rollback, transaction is already committed: " + tx); + switch (tx.state()) { + case COMMITTED: + if (log.isDebugEnabled()) + log.debug("Failed to rollback, transaction is already committed: " + tx); + + // Fall-through. - rollbackFut.forceFinish(); + case ROLLED_BACK: + rollbackFut.forceFinish(); + + assert rollbackFut.isDone() : rollbackFut; + + break; - assert rollbackFut.isDone() : rollbackFut; + default: // First finish attempt was unsuccessful. Try again. + rollbackFut.finish(false, clearThreadMap, onTimeout); } - else // First finish attempt was unsuccessful. Try again. - rollbackFut.finish(false, clearThreadMap, onTimeout); } else { finishFut.listen(new IgniteInClosure>() { From 10b98c55929b157290f9da7bcb361db001176cde Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Thu, 6 Dec 2018 18:51:24 +0300 Subject: [PATCH 213/403] IGNITE-10242 Pause ongoing rebalance on cache group stopping Signed-off-by: Pavel Kovalenko (cherry picked from commit 025a0360c86a5ceb8c3eef2f514c29846081d4c2) --- .../GridCachePartitionExchangeManager.java | 4 +- .../processors/cache/GridCachePreloader.java | 10 + .../cache/GridCachePreloaderAdapter.java | 10 + .../processors/cache/GridCacheProcessor.java | 106 ++++--- .../preloader/GridDhtPartitionDemander.java | 3 + .../GridDhtPartitionSupplyMessage.java | 3 + .../GridDhtPartitionsExchangeFuture.java | 8 +- .../GridDhtPartitionsFullMessage.java | 5 +- .../dht/preloader/GridDhtPreloader.java | 51 +++- .../ignite/internal/util/IgniteUtils.java | 66 +++++ ...lanceOnCachesStoppingOrDestroyingTest.java | 279 ++++++++++++++++++ .../testsuites/IgnitePdsTestSuite4.java | 3 + 12 files changed, 498 insertions(+), 50 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 81f8b8c8af204..81488e29ebf14 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -437,9 +437,11 @@ else if (m instanceof GridDhtPartitionDemandLegacyMessage) { return; } + else + U.error(log, "Unsupported message type: " + m.getClass().getName()); } - U.error(log, "Unsupported message type: " + m.getClass().getName()); + U.warn(log, "Cache group with id=" + m.groupId() + " is stopped or absent"); } finally { leaveBusy(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java index d629e94db7e84..6ac26c958fc29 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java @@ -192,4 +192,14 @@ public GridDhtFuture request(GridCacheContext cctx, * Dumps debug information. */ public void dumpDebugInfo(); + + /** + * Pause preloader. + */ + public void pause(); + + /** + * Resume preloader. + */ + public void resume(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java index c5e4a817d0418..f16305cd85456 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java @@ -181,4 +181,14 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) { @Override public void dumpDebugInfo() { // No-op. } + + /** {@inheritDoc} */ + @Override public void pause() { + // No-op + } + + /** {@inheritDoc} */ + @Override public void resume() { + // No-op + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index b69df8da44916..30a2a14f8ff8c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -74,6 +74,7 @@ import org.apache.ignite.internal.binary.BinaryContext; import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.binary.GridBinaryMarshaller; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -2093,10 +2094,8 @@ private void prepareStartCaches( else { Map cacheContexts = new ConcurrentHashMap<>(); - int parallelismLvl = sharedCtx.kernalContext().config().getSystemThreadPoolSize(); - // Reserve at least 2 threads for system operations. - parallelismLvl = Math.max(1, parallelismLvl - 2); + int parallelismLvl = U.availableThreadCount(ctx, GridIoPolicy.SYSTEM_POOL, 2); doInParallel( parallelismLvl, @@ -2876,7 +2875,9 @@ void forceCloseCaches(ExchangeActions exchActions) { * @param exchActions Change requests. */ private void processCacheStopRequestOnExchangeDone(ExchangeActions exchActions) { - // Force checkpoint if there is any cache stop request + // Reserve at least 2 threads for system operations. + int parallelismLvl = U.availableThreadCount(ctx, GridIoPolicy.SYSTEM_POOL, 2); + if (!exchActions.cacheStopRequests().isEmpty()) { try { sharedCtx.database().waitForCheckpoint("caches stop"); @@ -2886,63 +2887,88 @@ private void processCacheStopRequestOnExchangeDone(ExchangeActions exchActions) } } - for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) { - CacheGroupContext gctx = cacheGrps.get(action.descriptor().groupId()); + List> grpToStop = exchActions.cacheGroupsToStop().stream() + .filter(a -> cacheGrps.containsKey(a.descriptor().groupId())) + .map(a -> F.t(cacheGrps.get(a.descriptor().groupId()), a.destroy())) + .collect(Collectors.toList()); - // Cancel all operations blocking gateway - if (gctx != null) { - final String msg = "Failed to wait for topology update, cache group is stopping."; + Map> cachesToStop = exchActions.cacheStopRequests().stream() + .collect(Collectors.groupingBy(action -> action.descriptor().groupId())); - // If snapshot operation in progress we must throw CacheStoppedException - // for correct cache proxy restart. For more details see - // IgniteCacheProxy.cacheException() - gctx.affinity().cancelFutures(new CacheStoppedException(msg)); - } + try { + doInParallel( + parallelismLvl, + sharedCtx.kernalContext().getSystemExecutorService(), + cachesToStop.entrySet(), + cachesToStopByGrp -> { + CacheGroupContext gctx = cacheGrps.get(cachesToStopByGrp.getKey()); - stopGateway(action.request()); + if (gctx != null) + gctx.preloader().pause(); - sharedCtx.database().checkpointReadLock(); + try { - try { - prepareCacheStop(action.request().cacheName(), action.request().destroy()); - } - finally { - sharedCtx.database().checkpointReadUnlock(); - } + if (gctx != null) { + final String msg = "Failed to wait for topology update, cache group is stopping."; + + // If snapshot operation in progress we must throw CacheStoppedException + // for correct cache proxy restart. For more details see + // IgniteCacheProxy.cacheException() + gctx.affinity().cancelFutures(new CacheStoppedException(msg)); + } + + for (ExchangeActions.CacheActionData action: cachesToStopByGrp.getValue()) { + stopGateway(action.request()); + + sharedCtx.database().checkpointReadLock(); + + try { + prepareCacheStop(action.request().cacheName(), action.request().destroy()); + } + finally { + sharedCtx.database().checkpointReadUnlock(); + } + } + } + finally { + if (gctx != null) + gctx.preloader().resume(); + } + + return null; + } + ); + } + catch (IgniteCheckedException e) { + String msg = "Failed to stop caches"; + + log.error(msg, e); + + throw new IgniteException(msg, e); } sharedCtx.database().checkpointReadLock(); try { // Do not invoke checkpoint listeners for groups are going to be destroyed to prevent metadata corruption. - for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) { - Integer groupId = action.descriptor().groupId(); - CacheGroupContext grp = cacheGrps.get(groupId); + grpToStop.forEach(grp -> { + CacheGroupContext gctx = grp.getKey(); - if (grp != null && grp.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) { + if (gctx != null && gctx.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) { GridCacheDatabaseSharedManager mngr = (GridCacheDatabaseSharedManager)sharedCtx.database(); - mngr.removeCheckpointListener((DbCheckpointListener)grp.offheap()); + mngr.removeCheckpointListener((DbCheckpointListener)gctx.offheap()); } - } + }); } finally { sharedCtx.database().checkpointReadUnlock(); } - List> stoppedGroups = new ArrayList<>(); - - for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) { - Integer groupId = action.descriptor().groupId(); - - if (cacheGrps.containsKey(groupId)) { - stoppedGroups.add(F.t(cacheGrps.get(groupId), action.destroy())); - - stopCacheGroup(groupId); - } - } + for (IgniteBiTuple grp : grpToStop) + stopCacheGroup(grp.get1().groupId()); if (!sharedCtx.kernalContext().clientNode()) - sharedCtx.database().onCacheGroupsStopped(stoppedGroups); + sharedCtx.database().onCacheGroupsStopped(grpToStop); if (exchActions.deactivate()) sharedCtx.deactivate(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java index 40defa14b41e4..e24d956945af6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java @@ -890,6 +890,9 @@ private boolean preloadEntry( try { GridCacheContext cctx = grp.sharedGroup() ? ctx.cacheContext(entry.cacheId()) : grp.singleCacheContext(); + if (cctx == null) + return true; + if (cctx.isNear()) cctx = cctx.dhtCache().context(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java index 3034fb9daaf21..7e281e59a7e1e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java @@ -249,6 +249,9 @@ void addEntry0(int p, boolean historical, GridCacheEntryInfo info, GridCacheShar CacheGroupContext grp = ctx.cache().cacheGroup(grpId); + if (grp == null) + return; + for (CacheEntryInfoCollection col : infos().values()) { List entries = col.infos(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index eeaec4ec594db..1f6c9679d75fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3537,10 +3537,8 @@ private void assignPartitionsStates() { * failed to send update counter deltas to backup. */ private void finalizePartitionCounters() { - int parallelismLvl = cctx.kernalContext().config().getSystemThreadPoolSize(); - // Reserve at least 2 threads for system operations. - parallelismLvl = Math.max(1, parallelismLvl - 2); + int parallelismLvl = U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2); long time = System.currentTimeMillis(); @@ -3969,10 +3967,8 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa long time = System.currentTimeMillis(); - int parallelismLvl = cctx.kernalContext().config().getSystemThreadPoolSize(); - // Reserve at least 2 threads for system operations. - parallelismLvl = Math.max(1, parallelismLvl - 2); + int parallelismLvl = U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2); try { doInParallel( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index fbaa24193b50f..a2cecb8ba527c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -31,6 +31,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridDirectMap; import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; @@ -415,7 +416,7 @@ void setErrorsMap(Map errs) { if (marshal) { // Reserve at least 2 threads for system operations. - int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2); + int parallelismLvl = U.availableThreadCount(ctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2); Collection objectsToMarshall = new ArrayList<>(); @@ -509,7 +510,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { Collection objectsToUnmarshall = new ArrayList<>(); // Reserve at least 2 threads for system operations. - int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2); + int parallelismLvl = U.availableThreadCount(ctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2); if (partsBytes != null && parts == null) objectsToUnmarshall.add(partsBytes); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java index cb26c0dd30e88..b3f597cd5d5ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java @@ -20,13 +20,16 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Queue; import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.processors.affinity.AffinityAssignment; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; @@ -42,7 +45,9 @@ import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.GridPlainRunnable; +import org.apache.ignite.internal.util.lang.GridTuple3; import org.apache.ignite.internal.util.typedef.CI1; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgnitePredicate; import org.jetbrains.annotations.Nullable; @@ -79,6 +84,12 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter { /** Demand lock. */ private final ReadWriteLock demandLock = new ReentrantReadWriteLock(); + /** */ + private boolean paused; + + /** */ + private Queue> pausedDemanderQueue = new ConcurrentLinkedQueue<>(); + /** */ private boolean stopped; @@ -358,7 +369,10 @@ private List remoteOwners(int p, AffinityTopologyVersion topVer) { demandLock.readLock().lock(); try { - demander.handleSupplyMessage(idx, id, s); + if (paused) + pausedDemanderQueue.add(F.t(idx, id, s)); + else + demander.handleSupplyMessage(idx, id, s); } finally { demandLock.readLock().unlock(); @@ -563,6 +577,41 @@ private GridDhtFuture request0(GridCacheContext cctx, Collection> msgToProc = + new ArrayList<>(pausedDemanderQueue); + + pausedDemanderQueue.clear(); + + final GridDhtPreloader preloader = this; + + ctx.kernalContext().closure().runLocalSafe(() -> msgToProc.forEach( + m -> preloader.handleSupplyMessage(m.get1(), m.get2(), m.get3()) + ), GridIoPolicy.SYSTEM_POOL); + + paused = false; + } + finally { + demandLock.writeLock().unlock(); + } + } + /** {@inheritDoc} */ @Override public void dumpDebugInfo() { // No-op diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index fcb99782bbe46..e01f89960cc88 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -55,6 +55,7 @@ import org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException; import org.apache.ignite.internal.events.DiscoveryCustomEvent; import org.apache.ignite.internal.managers.communication.GridIoManager; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo; import org.apache.ignite.internal.mxbean.IgniteStandardMXBean; import org.apache.ignite.internal.processors.cache.CacheClassLoaderMarker; @@ -10613,6 +10614,71 @@ public static ClusterNode randomServerNode(GridKernalContext ctx) { return rndNode; } + /** + * @param ctx Kernal context. + * @param plc IO Policy. + * @param reserved Thread to reserve. + * @return Number of available threads in executor service for {@code plc}. If {@code plc} + * is invalid, return {@code 1}. + */ + public static int availableThreadCount(GridKernalContext ctx, byte plc, int reserved) { + IgniteConfiguration cfg = ctx.config(); + + int parallelismLvl; + + switch (plc) { + case GridIoPolicy.P2P_POOL: + parallelismLvl = cfg.getPeerClassLoadingThreadPoolSize(); + + break; + + case GridIoPolicy.SYSTEM_POOL: + parallelismLvl = cfg.getSystemThreadPoolSize(); + + break; + + case GridIoPolicy.PUBLIC_POOL: + parallelismLvl = cfg.getPublicThreadPoolSize(); + + break; + + case GridIoPolicy.MANAGEMENT_POOL: + parallelismLvl = cfg.getManagementThreadPoolSize(); + + break; + + case GridIoPolicy.UTILITY_CACHE_POOL: + parallelismLvl = cfg.getUtilityCacheThreadPoolSize(); + + break; + + case GridIoPolicy.IGFS_POOL: + parallelismLvl = cfg.getIgfsThreadPoolSize(); + + break; + + case GridIoPolicy.SERVICE_POOL: + parallelismLvl = cfg.getServiceThreadPoolSize(); + + break; + + case GridIoPolicy.DATA_STREAMER_POOL: + parallelismLvl = cfg.getDataStreamerThreadPoolSize(); + + break; + + case GridIoPolicy.QUERY_POOL: + parallelismLvl = cfg.getQueryThreadPoolSize(); + + break; + + default: + parallelismLvl = -1; + } + + return Math.max(1, parallelismLvl - reserved); + } + /** * Execute operation on data in parallel. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java new file mode 100644 index 0000000000000..97f8d455e77d4 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.rebalancing; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.failure.StopNodeFailureHandler; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * + */ +public class IgniteRebalanceOnCachesStoppingOrDestroyingTest extends GridCommonAbstractTest { + /** */ + private static final String CACHE_1 = "cache_1"; + + /** */ + private static final String CACHE_2 = "cache_2"; + + /** */ + private static final String CACHE_3 = "cache_3"; + + /** */ + private static final String CACHE_4 = "cache_4"; + + /** */ + private static final String GROUP_1 = "group_1"; + + /** */ + private static final String GROUP_2 = "group_2"; + + /** */ + private static final int REBALANCE_BATCH_SIZE = 50 * 1024; + + /** */ + private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + TcpDiscoverySpi spi = new TcpDiscoverySpi(); + + spi.setIpFinder(ipFinder); + + cfg.setDiscoverySpi(spi); + + cfg.setCommunicationSpi(new RebalanceBlockingSPI()); + + cfg.setFailureHandler(new StopNodeFailureHandler()); + + cfg.setRebalanceThreadPoolSize(4); + + cfg.setTransactionConfiguration(new TransactionConfiguration() + .setDefaultTxTimeout(1000)); + + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setWalMode(WALMode.LOG_ONLY) + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(100L * 1024 * 1024))); + + return cfg; + } + + /** + * + */ + public void testStopCachesOnDeactivation() throws Exception { + performTest(ig -> { + ig.cluster().active(false); + + // Add to escape possible long waiting in awaitPartitionMapExchange due to {@link CacheAffinityChangeMessage}. + ig.cluster().active(true); + + return null; + }); + } + + /** + * + */ + public void testDestroySpecificCachesInDifferentCacheGroups() throws Exception { + performTest(ig -> { + ig.destroyCaches(Arrays.asList(CACHE_1, CACHE_3)); + + return null; + }); + } + + /** + * + */ + public void testDestroySpecificCacheAndCacheGroup() throws Exception { + performTest(ig -> { + ig.destroyCaches(Arrays.asList(CACHE_1, CACHE_3, CACHE_4)); + + return null; + }); + } + + /** + * @param testAction Action that trigger stop or destroy of caches. + */ + private void performTest(IgniteThrowableConsumer testAction) throws Exception { + IgniteEx ig0 = (IgniteEx)startGrids(2); + + ig0.cluster().active(true); + + stopGrid(1); + + loadData(ig0); + + startGrid(1); + + runLoad(ig0); + + testAction.accept(ig0); + + U.sleep(1000); + + awaitPartitionMapExchange(true, true, null, true); + + assertNull(grid(1).context().failure().failureContext()); + } + + /** + * @param ig Ig. + */ + private void loadData(Ignite ig) { + List configs = Stream.of( + F.t(CACHE_1, GROUP_1), + F.t(CACHE_2, GROUP_1), + F.t(CACHE_3, GROUP_2), + F.t(CACHE_4, GROUP_2) + ).map(names -> new CacheConfiguration<>(names.get1()) + .setGroupName(names.get2()) + .setRebalanceBatchSize(REBALANCE_BATCH_SIZE) + .setCacheMode(CacheMode.REPLICATED) + ).collect(Collectors.toList()); + + ig.getOrCreateCaches(configs); + + configs.forEach(cfg -> { + try (IgniteDataStreamer streamer = ig.dataStreamer(cfg.getName())) { + for (int i = 0; i < 3_000; i++) + streamer.addData(i, new byte[1024]); + + streamer.flush(); + } + }); + } + + /** + * @param ig Ignite instance. + */ + private void runLoad(Ignite ig) throws Exception{ + GridTestUtils.runMultiThreaded(new Runnable() { + @Override public void run() { + String cacheName = F.rand(CACHE_1, CACHE_2, CACHE_3, CACHE_4); + + IgniteCache cache = ig.cache(cacheName); + + for (int i = 0; i < 3_000; i++) { + int idx = ThreadLocalRandom.current().nextInt(3_000); + + cache.put(idx, new byte[1024]); + } + } + }, 4, "load-thread"); + } + + /** + * + */ + private static class RebalanceBlockingSPI extends TcpCommunicationSpi { + /** */ + public static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + slowDownMessage(msg); + + super.sendMessage(node, msg); + + } + + /** {@inheritDoc} */ + @Override public void sendMessage(ClusterNode node, Message msg, + IgniteInClosure ackC) throws IgniteSpiException { + slowDownMessage(msg); + + super.sendMessage(node, msg, ackC); + } + + /** + * @param msg Message. + */ + private void slowDownMessage(Message msg) { + if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) { + int grpId = ((GridCacheGroupIdMessage)((GridIoMessage)msg).message()).groupId(); + + if (grpId == CU.cacheId(GROUP_1) || grpId == CU.cacheId(GROUP_2)) { + try { + U.sleep(50); + } + catch (IgniteInterruptedCheckedException e) { + e.printStackTrace(); + } + } + } + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java index ad6bbbf0ec2b6..5e6d45a1fb8ed 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java @@ -20,6 +20,7 @@ import junit.framework.TestSuite; import org.apache.ignite.cache.ResetLostPartitionTest; import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse; +import org.apache.ignite.internal.processors.cache.distributed.rebalancing.IgniteRebalanceOnCachesStoppingOrDestroyingTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest; @@ -51,6 +52,8 @@ public static TestSuite suite() { suite.addTestSuite(ResetLostPartitionTest.class); + suite.addTestSuite(IgniteRebalanceOnCachesStoppingOrDestroyingTest.class); + suite.addTestSuite(IgnitePdsCacheWalDisabledOnRebalancingTest.class); return suite; From 13f221edacfd72266e0ea4ad0161bfde50180eff Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 7 Dec 2018 14:07:28 +0700 Subject: [PATCH 214/403] IGNITE-10552 Web Agent: Improve logging when cluster topology changed. (cherry picked from commit 4884dec6c8c2e3698d289e4be6c177477d091adc) --- .../agent/handlers/ClusterListener.java | 10 ++++++++ .../console/agent/rest/RestExecutor.java | 24 ++++++++++++------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java index 6985837e1044e..14d3d5d7835c8 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java @@ -368,6 +368,14 @@ Collection nid8() { boolean differentCluster(TopologySnapshot prev) { return prev == null || F.isEmpty(prev.nids) || Collections.disjoint(nids, prev.nids); } + + /** + * @param prev Previous topology. + * @return {@code true} in case if current topology is the same cluster, but topology changed. + */ + boolean topologyChanged(TopologySnapshot prev) { + return prev != null && !prev.nids.equals(nids); + } } /** */ @@ -495,6 +503,8 @@ public boolean active(IgniteProductVersion ver, UUID nid) throws IOException { if (newTop.differentCluster(top)) log.info("Connection successfully established to cluster with nodes: " + newTop.nid8()); + else if (newTop.topologyChanged(top)) + log.info("Cluster topology changed, new topology: " + newTop.nid8()); boolean active = active(newTop.clusterVersion(), F.first(newTop.getNids())); diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java index 737b966332edd..d3bdcdd6e1b87 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java @@ -141,10 +141,11 @@ private RestResult parseResponse(Response res) throws IOException { /** */ private RestResult sendRequest(String url, Map params, Map headers) throws IOException { - HttpUrl httpUrl = HttpUrl.parse(url); - - HttpUrl.Builder urlBuilder = httpUrl.newBuilder() - .addPathSegment("ignite"); + HttpUrl httpUrl = HttpUrl + .parse(url) + .newBuilder() + .addPathSegment("ignite") + .build(); final Request.Builder reqBuilder = new Request.Builder(); @@ -163,8 +164,7 @@ private RestResult sendRequest(String url, Map params, Map params, Map nodeURIs, Map params, Map headers) throws IOException { Integer startIdx = startIdxs.getOrDefault(nodeURIs, 0); - for (int i = 0; i < nodeURIs.size(); i++) { - Integer currIdx = (startIdx + i) % nodeURIs.size(); + int urlsCnt = nodeURIs.size(); + + for (int i = 0; i < urlsCnt; i++) { + Integer currIdx = (startIdx + i) % urlsCnt; String nodeUrl = nodeURIs.get(currIdx); try { RestResult res = sendRequest(nodeUrl, params, headers); + // If first attempt failed then throttling should be cleared. + if (i > 0) + LT.clear(); + LT.info(log, "Connected to cluster [url=" + nodeUrl + "]"); startIdxs.put(nodeURIs, currIdx); @@ -190,7 +196,7 @@ public RestResult sendRequest(List nodeURIs, Map params, return res; } catch (ConnectException ignored) { - // No-op. + LT.warn(log, "Failed to connect to cluster [url=" + nodeUrl + "]"); } } From d16e3466330712e16d743705308bca1fa75d3653 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Fri, 7 Dec 2018 16:26:00 +0700 Subject: [PATCH 215/403] IGNITE-10318 Web console: Update panels to use newer UI on "Query notebook" screen. (cherry picked from commit e1b8686aa9d7887d462e0f6b04f33501a9f31141) --- .../input-dialog/input-dialog.tpl.pug | 1 - .../query-actions-button/component.ts | 28 ++++ .../query-actions-button/controller.ts | 53 +++++++ .../query-actions-button/template.pug | 28 ++++ .../{controller.js => controller.ts} | 143 ++++++++++++------ .../components/queries-notebook/index.js | 2 + .../queries-notebook/template.tpl.pug | 50 +++--- 7 files changed, 231 insertions(+), 74 deletions(-) create mode 100644 modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/component.ts create mode 100644 modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/controller.ts create mode 100644 modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/template.pug rename modules/web-console/frontend/app/components/page-queries/components/queries-notebook/{controller.js => controller.ts} (93%) diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug index d79cb565e3294..23896e78d5840 100644 --- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug +++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug @@ -21,7 +21,6 @@ include /app/helpers/jade/mixins form.modal-content(name='ctrl.form' novalidate) .modal-header h4.modal-title - i.fa.fa-clone span {{ ctrl.options.title }} button.close(type='button' aria-label='Close' ng-click='$hide()') svg(ignite-icon="cross") diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/component.ts b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/component.ts new file mode 100644 index 0000000000000..099efc20aba90 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/component.ts @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import QueryActionsButton from './controller'; + +export const component: ng.IComponentOptions = { + controller: QueryActionsButton, + template, + bindings: { + actions: '<', + item: '<' + } +}; diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/controller.ts b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/controller.ts new file mode 100644 index 0000000000000..e71cf4603dbb2 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/controller.ts @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type QueryActions < T > = Array<{text: string, click?(item: T): any, available?(item: T): boolean}>; + +export default class QueryActionButton { + static $inject = ['$element']; + + item: T; + + actions: QueryActions; + + boundActions: QueryActions = []; + + constructor(private el: JQLite) {} + + $postLink() { + this.el[0].classList.add('btn-ignite-group'); + } + + $onChanges(changes: {actions: ng.IChangesObject['actions']>}) { + if ('actions' in changes) { + this.boundActions = changes.actions.currentValue.map((a) => { + const action = {...a}; + + const click = () => a.click(this.item); + + Object.defineProperty(action, 'click', { + get: () => { + return typeof a.available === 'function' + ? a.available(this.item) ? click : void 0 + : a.available ? click : void 0; + } + }); + return action; + }); + } + } +} diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/template.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/template.pug new file mode 100644 index 0000000000000..32359bf3a4ed6 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/components/query-actions-button/template.pug @@ -0,0 +1,28 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +button.btn-ignite.btn-ignite--primary( + ng-click='$ctrl.boundActions[0].click()' + type='button' + ng-disabled='!$ctrl.boundActions[0].click' +) + | {{ ::$ctrl.boundActions[0].text }} +button.btn-ignite.btn-ignite--primary( + bs-dropdown='$ctrl.boundActions' + data-placement='bottom-right' + type='button' +) + span.icon.fa.fa-caret-down diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts similarity index 93% rename from modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js rename to modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts index 4b1b8611a67fe..3ac3177a7e2ee 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts @@ -35,6 +35,12 @@ import cacheMetadataTemplateUrl from 'views/sql/cache-metadata.tpl.pug'; import chartSettingsTemplateUrl from 'views/sql/chart-settings.tpl.pug'; import messageTemplateUrl from 'views/templates/message.tpl.pug'; +import {default as Notebook} from '../../notebook.service'; +import {default as MessagesServiceFactory} from 'app/services/Messages.service'; +import {default as LegacyConfirmServiceFactory} from 'app/services/Confirm.service'; +import {default as InputDialog} from 'app/components/input-dialog/input-dialog.service'; +import {QueryActions} from './components/query-actions-button/controller'; + // Time line X axis descriptor. const TIME_LINE = {value: -1, type: 'java.sql.Date', label: 'TIME_LINE'}; @@ -68,6 +74,9 @@ const _fullColName = (col) => { let paragraphId = 0; class Paragraph { + name: string; + qryType: 'scan' | 'query'; + constructor($animate, $timeout, JavaTypes, errorParser, paragraph) { const self = this; @@ -253,12 +262,12 @@ class Paragraph { // Controller for SQL notebook screen. export class NotebookCtrl { - static $inject = ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'AgentManager', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes', 'IgniteCopyToClipboard', 'CSV', 'IgniteErrorParser', 'DemoInfo']; + static $inject = ['IgniteInput', '$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'AgentManager', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes', 'IgniteCopyToClipboard', 'CSV', 'IgniteErrorParser', 'DemoInfo']; /** * @param {CSV} CSV */ - constructor($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, IgniteCopyToClipboard, CSV, errorParser, DemoInfo) { + constructor(private IgniteInput: InputDialog, $root, private $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, private Messages: ReturnType, private Confirm: ReturnType, agentMgr, IgniteChartColors, private Notebook: Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, IgniteCopyToClipboard, CSV, errorParser, DemoInfo) { const $ctrl = this; this.CSV = CSV; @@ -1005,23 +1014,6 @@ export class NotebookCtrl { $scope.removeNotebook = (notebook) => Notebook.remove(notebook); - $scope.renameParagraph = function(paragraph, newName) { - if (!newName) - return; - - if (paragraph.name !== newName) { - paragraph.name = newName; - - $scope.rebuildScrollParagraphs(); - - Notebook.save($scope.notebook) - .then(() => paragraph.edit = false) - .catch(Messages.showError); - } - else - paragraph.edit = false; - }; - $scope.addParagraph = (paragraph, sz) => { if ($scope.caches && $scope.caches.length > 0) paragraph.cacheName = _.head($scope.caches).value; @@ -1125,31 +1117,6 @@ export class NotebookCtrl { return (paragraph.result === result); }; - $scope.removeParagraph = function(paragraph) { - Confirm.confirm('Are you sure you want to remove query: "' + paragraph.name + '"?') - .then(function() { - $scope.stopRefresh(paragraph); - - const paragraph_idx = _.findIndex($scope.notebook.paragraphs, function(item) { - return paragraph === item; - }); - - const panel_idx = _.findIndex($scope.expandedParagraphs, function(item) { - return paragraph_idx === item; - }); - - if (panel_idx >= 0) - $scope.expandedParagraphs.splice(panel_idx, 1); - - $scope.notebook.paragraphs.splice(paragraph_idx, 1); - - $scope.rebuildScrollParagraphs(); - - Notebook.save($scope.notebook) - .catch(Messages.showError); - }); - }; - $scope.paragraphExpanded = function(paragraph) { const paragraph_idx = _.findIndex($scope.notebook.paragraphs, function(item) { return paragraph === item; @@ -1945,6 +1912,94 @@ export class NotebookCtrl { }; } + scanActions: QueryActions = [ + { + text: 'Scan', + click: (p) => this.$scope.scan(p), + available: (p) => this.$scope.scanAvailable(p) + }, + { + text: 'Scan on selected node', + click: (p) => this.$scope.scan(p, true), + available: (p) => this.$scope.scanAvailable(p) + }, + {text: 'Rename', click: (p) => this.renameParagraph(p), available: () => true}, + {text: 'Remove', click: (p) => this.removeParagraph(p), available: () => true} + ]; + + queryActions: QueryActions = [ + { + text: 'Execute', + click: (p) => this.$scope.execute(p), + available: (p) => this.$scope.queryAvailable(p) + }, + { + text: 'Execute on selected node', + click: (p) => this.$scope.execute(p, true), + available: (p) => this.$scope.queryAvailable(p) + }, + { + text: 'Explain', + click: (p) => this.$scope.explain(p), + available: (p) => this.$scope.queryAvailable(p) + }, + {text: 'Rename', click: (p) => this.renameParagraph(p), available: () => true}, + {text: 'Remove', click: (p) => this.removeParagraph(p), available: () => true} + ]; + + async renameParagraph(paragraph: Paragraph) { + try { + const newName = await this.IgniteInput.input('Rename Query', 'New query name:', paragraph.name); + + if (paragraph.name !== newName) { + paragraph.name = newName; + + this.$scope.rebuildScrollParagraphs(); + + await this.Notebook.save(this.$scope.notebook) + .catch(this.Messages.showError); + } + } + catch (ignored) { + // No-op. + } + } + + async removeParagraph(paragraph: Paragraph) { + try { + await this.Confirm.confirm('Are you sure you want to remove query: "' + paragraph.name + '"?'); + this.$scope.stopRefresh(paragraph); + + const paragraph_idx = _.findIndex(this.$scope.notebook.paragraphs, (item) => paragraph === item); + const panel_idx = _.findIndex(this.$scope.expandedParagraphs, (item) => paragraph_idx === item); + + if (panel_idx >= 0) + this.$scope.expandedParagraphs.splice(panel_idx, 1); + + this.$scope.notebook.paragraphs.splice(paragraph_idx, 1); + this.$scope.rebuildScrollParagraphs(); + + await this.Notebook.save(this.$scope.notebook) + .catch(this.Messages.showError); + } + catch (ignored) { + // No-op. + } + } + + isParagraphOpened(index: number) { + return this.$scope.notebook.expandedParagraphs.includes(index); + } + + onParagraphClose(index: number) { + const expanded = this.$scope.notebook.expandedParagraphs; + expanded.splice(expanded.indexOf(index), 1); + } + + onParagraphOpen(index: number) { + this.$scope.notebook.expandedParagraphs.push(index); + } + $onDestroy() { if (this.refresh$) this.refresh$.unsubscribe(); diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/index.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/index.js index bf93f42586d68..57aa779c05019 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/index.js +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/index.js @@ -19,9 +19,11 @@ import angular from 'angular'; import templateUrl from './template.tpl.pug'; import { NotebookCtrl } from './controller'; import NotebookData from '../../notebook.data'; +import {component as actions} from './components/query-actions-button/component'; import './style.scss'; export default angular.module('ignite-console.sql.notebook', []) + .component('queryActionsButton', actions) .component('queriesNotebook', { controller: NotebookCtrl, templateUrl diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug index af673a822d9fb..e2c440882bb6f 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug @@ -71,24 +71,6 @@ mixin notebook-error label.col-sm-12 Notebook not accessible any more. Go back to notebooks list. button.h3.btn.btn-primary(ui-sref='default-state') Leave Notebook -mixin paragraph-rename - .col-sm-6(ng-hide='paragraph.edit') - i.fa(ng-class='paragraphExpanded(paragraph) ? "fa-chevron-circle-down" : "fa-chevron-circle-right"') - label {{paragraph.name}} - - .btn-group(ng-hide='notebook.paragraphs.length > 1') - +btn-toolbar('fa-pencil', 'paragraph.edit = true; paragraph.editName = paragraph.name; $event.stopPropagation();', 'Rename query', 'paragraph-name-{{paragraph.id}}') - - .btn-group(ng-show='notebook.paragraphs.length > 1' ng-click='$event.stopPropagation();') - +btn-toolbar('fa-pencil', 'paragraph.edit = true; paragraph.editName = paragraph.name;', 'Rename query', 'paragraph-name-{{paragraph.id}}') - +btn-toolbar('fa-remove', 'removeParagraph(paragraph)', 'Remove query') - - .col-sm-6(ng-show='paragraph.edit') - i.tipLabel.fa(style='float: left;' ng-class='paragraphExpanded(paragraph) ? "fa-chevron-circle-down" : "fa-chevron-circle-right"') - i.tipLabel.fa.fa-floppy-o(style='float: right;' ng-show='paragraph.editName' ng-click='renameParagraph(paragraph, paragraph.editName); $event.stopPropagation();' bs-tooltip data-title='Save query name' data-trigger='hover') - .input-tip - input.form-control(id='paragraph-name-{{paragraph.id}}' ng-model='paragraph.editName' required ng-click='$event.stopPropagation();' ignite-on-enter='renameParagraph(paragraph, paragraph.editName)' ignite-on-escape='paragraph.edit = false') - mixin query-settings div .form-field--inline( @@ -314,10 +296,10 @@ mixin chart-result label.margin-top-dflt Charts do not support #[b Explain] and #[b Scan] query mixin paragraph-scan - .panel-heading(bs-collapse-toggle) - .row - +paragraph-rename - .panel-collapse(role='tabpanel' bs-collapse-target) + panel-title {{ paragraph.name }} + panel-actions + query-actions-button(actions='$ctrl.scanActions' item='paragraph') + panel-content .col-sm-12.sql-controls .col-sm-3 +form-field__dropdown({ @@ -378,9 +360,10 @@ mixin paragraph-scan a Next mixin paragraph-query - .row.panel-heading(bs-collapse-toggle) - +paragraph-rename - .panel-collapse.ng-animate-disabled(role='tabpanel' bs-collapse-target) + panel-title {{ paragraph.name }} + panel-actions + query-actions-button(actions='$ctrl.queryActions' item='paragraph') + panel-content .col-sm-12 .col-xs-8.col-sm-9(style='border-right: 1px solid #eee') .sql-editor(ignite-ace='{onLoad: aceInit(paragraph), theme: "chrome", mode: "sql", require: ["ace/ext/language_tools"],' + @@ -510,10 +493,19 @@ div div(ng-if='notebook' ignite-loading='sqlLoading' ignite-loading-text='{{ loadingText }}' ignite-loading-position='top') .docs-body.paragraphs - .panel-group(bs-collapse ng-model='notebook.expandedParagraphs' data-allow-multiple='true' data-start-collapsed='false') - + .panel-group .panel-paragraph(ng-repeat='paragraph in notebook.paragraphs' id='{{paragraph.id}}' ng-form='form_{{paragraph.id}}') - .panel.panel-default(ng-if='paragraph.qryType === "scan"') + panel-collapsible( + ng-if='paragraph.qryType === "scan"' + opened='$ctrl.isParagraphOpened($index)' + on-close='$ctrl.onParagraphClose($index)' + on-open='$ctrl.onParagraphOpen($index)' + ) +paragraph-scan - .panel.panel-default(ng-if='paragraph.qryType === "query"') + panel-collapsible( + ng-if='paragraph.qryType === "query"' + opened='$ctrl.isParagraphOpened($index)' + on-close='$ctrl.onParagraphClose($index)' + on-open='$ctrl.onParagraphOpen($index)' + ) +paragraph-query From 5664a03996bc57dd3c2c1ef2d1eee4bc8963c68f Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Fri, 7 Dec 2018 16:29:08 +0700 Subject: [PATCH 216/403] IGNITE-10587 Web Console: Fixed chart history cleanup. (cherry picked from commit 26860c5a9a9375baba81bcf67d140a4f4055d838) --- .../frontend/app/components/ignite-chart/controller.js | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/web-console/frontend/app/components/ignite-chart/controller.js b/modules/web-console/frontend/app/components/ignite-chart/controller.js index 81ed273f6882a..b6112502559c1 100644 --- a/modules/web-console/frontend/app/components/ignite-chart/controller.js +++ b/modules/web-console/frontend/app/components/ignite-chart/controller.js @@ -87,6 +87,7 @@ export class IgniteChartController { if ((changes.chartDataPoint && _.isNil(changes.chartDataPoint.currentValue)) || (changes.chartHistory && _.isEmpty(changes.chartHistory.currentValue))) { this.clearDatasets(); + this.localHistory = []; return; } From 80b43350a5d1bdabc0c2d01c95733094e24865fd Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Mon, 10 Dec 2018 09:17:48 +0700 Subject: [PATCH 217/403] IGNITE-10609 Web Console: Fixed removing cache from service on "Configuration" screen. (cherry picked from commit c7fe467fb235e3669f3ba4fb3007abb98dbb1e47) --- .../components/cluster-edit-form/controller.js | 7 ++++++- .../components/cluster-edit-form/templates/service.pug | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js index 881e5430378b7..5020d7bd18a56 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js @@ -25,6 +25,8 @@ export default class ClusterEditFormController { caches; /** @type {ig.menu} */ cachesMenu; + /** @type {ig.menu} */ + servicesCachesMenu; /** @type {ng.ICompiledExpression} */ onSave; @@ -108,8 +110,11 @@ export default class ClusterEditFormController { this.$scope.ui.inputForm.$setUntouched(); } } - if ('caches' in changes) + + if ('caches' in changes) { this.cachesMenu = (changes.caches.currentValue || []).map((c) => ({label: c.name, value: c._id})); + this.servicesCachesMenu = [{label: 'Key-affinity not used', value: null}].concat(this.cachesMenu); + } } /** diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug index 10b5dd8385324..6a2f8a60d1af5 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug @@ -83,9 +83,9 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`) label: 'Cache:', model: '$item.cache', name: '"serviceCache"', - placeholder: 'Choose cache', + placeholder: 'Key-affinity not used', placeholderEmpty: 'No caches configured for current cluster', - options: '$ctrl.cachesMenu', + options: '$ctrl.servicesCachesMenu', tip: 'Cache name used for key-to-node affinity calculation' })( pc-is-in-collection='$ctrl.clonedCluster.caches' From 433c899c1c0761f3fffbf880f3131feee58dedcd Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Mon, 10 Dec 2018 09:57:33 +0700 Subject: [PATCH 218/403] IGNITE-10610 Changed log from "error" to "warning" in VisorMultiNodeTask when no mapped jobs found. (cherry picked from commit ea63af0a713110222ba553a323cd152b8f78373c) --- .../org/apache/ignite/internal/visor/VisorMultiNodeTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java index 29ad5195c59e1..dbf1c6630905d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java @@ -105,7 +105,7 @@ protected Map map0(List subgrid, map.put(job(taskArg), node); if (map.isEmpty()) - ignite.log().error("No mapped jobs: [task=" + getClass().getName() + + ignite.log().warning("No mapped jobs: [task=" + getClass().getName() + ", topVer=" + ignite.cluster().topologyVersion() + ", jobNids=" + nodeIds + ", subGrid=" + U.toShortString(subgrid) + "]"); From 3c80e6cb4cd7fbe06569590aa87534a501b1eb65 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Mon, 10 Dec 2018 10:34:26 +0700 Subject: [PATCH 219/403] IgniteIGNITE-10611 Web Console: Refactor "ActivitiesService" to use uer object instead of id. (cherry picked from commit b131da0d60e9bf1e66269d0c8e38eecc47971559) --- modules/web-console/backend/routes/activities.js | 4 ++-- modules/web-console/backend/services/activities.js | 8 +++++--- modules/web-console/frontend/app/data/i18n.js | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/modules/web-console/backend/routes/activities.js b/modules/web-console/backend/routes/activities.js index 4db6ac2372ad0..11fd81bec75b6 100644 --- a/modules/web-console/backend/routes/activities.js +++ b/modules/web-console/backend/routes/activities.js @@ -24,8 +24,8 @@ const express = require('express'); module.exports = { implements: 'routes/activities', inject: ['services/activities'], + /** - * @param express * @param {ActivitiesService} activitiesService * @returns {Promise} */ @@ -35,7 +35,7 @@ module.exports = { // Post user activities to page. router.post('/page', (req, res) => { - activitiesService.merge(req.user._id, req.body) + activitiesService.merge(req.user, req.body) .then(res.api.ok) .catch(res.api.error); }); diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js index 9399eb056d8a6..bc0245de8f5e8 100644 --- a/modules/web-console/backend/services/activities.js +++ b/modules/web-console/backend/services/activities.js @@ -35,13 +35,15 @@ module.exports.factory = (mongo) => { /** * Update page activities. * - * @param {String} owner - User ID + * @param {Object} user - User. * @param {String} action - Action string presentation. * @param {String} group - Action group string presentation. * @param {Date} [now] - Optional date to save in activity. - * @returns {Promise.} that resolve activity + * @returns {Promise.} that resolve activity. */ - static merge(owner, {action, group}, now = new Date()) { + static merge(user, {action, group}, now = new Date()) { + const owner = user._id; + mongo.Account.findById(owner) .then((user) => { user.lastActivity = new Date(); diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js index a3c2480bdc724..20f9d25b2f338 100644 --- a/modules/web-console/frontend/app/data/i18n.js +++ b/modules/web-console/frontend/app/data/i18n.js @@ -46,10 +46,10 @@ export default { 'base.configuration.edit.advanced.models.model': 'Advanced cluster model edit', 'base.configuration.edit.advanced.igfs': 'Advanced cluster IGFSs', 'base.configuration.edit.advanced.igfs.igfs': 'Advanced cluster IGFS edit', - 'base.sql.tabs.notebooks-list': 'Query notebooks', - 'base.sql.notebook': 'Query notebook', - 'base.settings.profile': 'User profile', 'base.settings.admin': 'Admin panel', + 'base.settings.profile': 'User profile', + 'base.sql.notebook': 'Query notebook', + 'base.sql.tabs.notebooks-list': 'Query notebooks', // app/components/page-signin/template.pug 'app.components.page-signin.m1': 'Sign In', From 6a1871474872dd50ce7d40b5bd6d1fc6be6f3454 Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Mon, 10 Dec 2018 10:58:52 +0700 Subject: [PATCH 220/403] IGNITE-10565 Web Console: Add missing "maxRowsToShow" binding for "ui-grid" component. (cherry picked from commit 7f3f6b5d68830b3b27aa47e4737bfb17c1684839) --- modules/web-console/frontend/app/components/ui-grid/component.js | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/web-console/frontend/app/components/ui-grid/component.js b/modules/web-console/frontend/app/components/ui-grid/component.js index f0c3b06309055..d75cc6c2f1637 100644 --- a/modules/web-console/frontend/app/components/ui-grid/component.js +++ b/modules/web-console/frontend/app/components/ui-grid/component.js @@ -30,6 +30,7 @@ export default { gridHeight: ' Date: Mon, 10 Dec 2018 10:23:20 +0300 Subject: [PATCH 221/403] IGNITE-10589 Correctly handle exchanges merge when calculating last affinity change topology version - Fixes #5609. Signed-off-by: Alexey Goncharuk (cherry picked from commit d9be16f23f4df6405ea7e523c255dbe44e21c853) --- .../discovery/GridDiscoveryManager.java | 13 ++- .../affinity/AffinityTopologyVersion.java | 9 ++ .../GridCachePartitionExchangeManager.java | 14 +++- .../cache/CacheNoAffinityExchangeTest.java | 84 +++++++++++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index 75558380ea226..a9865f5ef2ea7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -2067,10 +2067,17 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer) AffinityTopologyVersion lastAffChangedTopVer = ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer); - DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer); + if (!lastAffChangedTopVer.equals(topVer)) { + assert lastAffChangedTopVer.compareTo(topVer) < 0; - if (lastAffChangedDiscoCache != null) - return lastAffChangedDiscoCache; + for (Map.Entry e : discoCacheHist.descendingEntrySet()) { + if (e.getKey().isBetween(lastAffChangedTopVer, topVer)) + return e.getValue(); + + if (e.getKey().compareTo(lastAffChangedTopVer) < 0) + break; + } + } CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java index 333841d1b33be..2c02f26be6641 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java @@ -112,6 +112,15 @@ public int minorTopologyVersion() { return cmp; } + /** + * @param lower Lower bound. + * @param upper Upper bound. + * @return {@code True} if this topology version is within provided bounds (inclusive). + */ + public boolean isBetween(AffinityTopologyVersion lower, AffinityTopologyVersion upper) { + return compareTo(lower) >= 0 && compareTo(upper) <= 0; + } + /** {@inheritDoc} */ @Override public void onAckReceived() { // No-op. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 81488e29ebf14..5a93b5acdb3e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -2849,12 +2849,18 @@ else if (task instanceof ForceRebalanceExchangeTask) { if (lastFut != null) { if (!lastFut.changedAffinity()) { - AffinityTopologyVersion lastAffVer = cctx.exchange().lastAffinityChangedTopologyVersion(lastFut.initialVersion()); - - cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastAffVer); + // If lastFut corresponds to merged exchange, it is essential to use + // topologyVersion() instead of initialVersion() - nodes joined in this PME + // will have DiscoCache only for the last version. + AffinityTopologyVersion lastAffVer = cctx.exchange() + .lastAffinityChangedTopologyVersion(lastFut.topologyVersion()); + + cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), + lastAffVer); } else - cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastFut.initialVersion()); + cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), + lastFut.topologyVersion()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java index 15cd5eeb0a14d..7eada30328eb0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java @@ -23,10 +23,19 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; @@ -63,6 +72,8 @@ public class CacheNoAffinityExchangeTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + cfg.setDiscoverySpi(new TestDiscoverySpi().setIpFinder(IP_FINDER)); if (startClient) { @@ -198,6 +209,79 @@ public void testNoAffinityChangeOnClientLeft() throws Exception { latch.countDown(); } + /** + * @throws Exception If failed. + */ + public void testNoAffinityChangeOnClientLeftWithMergedExchanges() throws Exception { + System.setProperty(IgniteSystemProperties.IGNITE_EXCHANGE_MERGE_DELAY, "1000"); + + try { + Ignite ig = startGridsMultiThreaded(4); + + IgniteCache atomicCache = ig.createCache(new CacheConfiguration() + .setName("atomic").setAtomicityMode(CacheAtomicityMode.ATOMIC).setCacheMode(CacheMode.REPLICATED)); + + IgniteCache txCache = ig.createCache(new CacheConfiguration() + .setName("tx").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).setCacheMode(CacheMode.REPLICATED)); + + startClient = true; + + Ignite client = startGrid("client"); + + startClient = false; + + stopGrid(1); + stopGrid(2); + stopGrid(3); + + awaitPartitionMapExchange(); + + atomicCache.put(-1, -1); + txCache.put(-1, -1); + + TestRecordingCommunicationSpi.spi(ig).blockMessages(new IgniteBiPredicate() { + @Override public boolean apply(ClusterNode node, Message message) { + return message instanceof GridDhtPartitionSupplyMessageV2; + } + }); + + startGridsMultiThreaded(1, 3); + + CountDownLatch latch = new CountDownLatch(1); + for (Ignite ignite : G.allGrids()) { + if (ignite.cluster().localNode().order() == 9) { + TestDiscoverySpi discoSpi = + (TestDiscoverySpi)((IgniteEx)ignite).context().discovery().getInjectedDiscoverySpi(); + + discoSpi.latch = latch; + + break; + } + } + + client.close(); + + for (int k = 0; k < 100; k++) { + atomicCache.put(k, k); + txCache.put(k, k); + + Lock lock = txCache.lock(k); + lock.lock(); + lock.unlock(); + } + + for (int k = 0; k < 100; k++) { + assertEquals(Integer.valueOf(k), atomicCache.get(k)); + assertEquals(Integer.valueOf(k), txCache.get(k)); + } + + latch.countDown(); + } + finally { + System.clearProperty(IgniteSystemProperties.IGNITE_EXCHANGE_MERGE_DELAY); + } + } + /** * */ From 159d0721152c7a6906bc9bfe9e4b7614b209950c Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Mon, 10 Dec 2018 15:58:14 +0300 Subject: [PATCH 222/403] IGNITE-8626: Added OpenSSL 1.1 support for C++ This closes #5590 (cherry picked from commit 4c48ae0ad29c6ce33868e7394041480da255771b) --- .../cpp/common/include/ignite/common/utils.h | 56 ++ modules/platforms/cpp/odbc/Makefile.am | 3 +- .../platforms/cpp/odbc/include/Makefile.am | 2 +- .../odbc/include/ignite/odbc/ssl/ssl_api.h | 32 + .../include/ignite/odbc/ssl/ssl_bindings.h | 357 ---------- .../include/ignite/odbc/ssl/ssl_gateway.h | 117 +++- .../cpp/odbc/project/vs/odbc.vcxproj | 3 +- .../cpp/odbc/project/vs/odbc.vcxproj.filters | 9 +- modules/platforms/cpp/odbc/src/connection.cpp | 4 +- .../cpp/odbc/src/ssl/secure_socket_client.cpp | 158 ++--- .../platforms/cpp/odbc/src/ssl/ssl_api.cpp | 33 + .../cpp/odbc/src/ssl/ssl_gateway.cpp | 639 +++++++++++++++--- .../cpp/thin-client-test/src/ssl_test.cpp | 2 +- modules/platforms/cpp/thin-client/Makefile.am | 1 + .../project/vs/thin-client.vcxproj | 5 +- .../project/vs/thin-client.vcxproj.filters | 11 +- .../cpp/thin-client/src/impl/data_channel.cpp | 4 +- .../src/impl/ssl/secure_socket_client.cpp | 139 ++-- .../cpp/thin-client/src/impl/ssl/ssl_api.cpp | 36 + .../cpp/thin-client/src/impl/ssl/ssl_api.h | 36 + .../thin-client/src/impl/ssl/ssl_bindings.h | 360 ---------- .../thin-client/src/impl/ssl/ssl_gateway.cpp | 632 ++++++++++++++--- .../thin-client/src/impl/ssl/ssl_gateway.h | 120 ++++ 23 files changed, 1718 insertions(+), 1041 deletions(-) create mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_api.h delete mode 100644 modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_bindings.h create mode 100644 modules/platforms/cpp/odbc/src/ssl/ssl_api.cpp create mode 100644 modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.cpp create mode 100644 modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.h delete mode 100644 modules/platforms/cpp/thin-client/src/impl/ssl/ssl_bindings.h diff --git a/modules/platforms/cpp/common/include/ignite/common/utils.h b/modules/platforms/cpp/common/include/ignite/common/utils.h index 29e36f525d90b..cbcc10ce6ab58 100644 --- a/modules/platforms/cpp/common/include/ignite/common/utils.h +++ b/modules/platforms/cpp/common/include/ignite/common/utils.h @@ -548,6 +548,62 @@ namespace ignite return BoundInstance(instance, mfunc); } + /** + * Method guard class template. + * + * Upon destruction calls provided method on provided class instance. + * + * @tparam T Value type. + */ + template + class MethodGuard + { + public: + /** Value type. */ + typedef T ValueType; + + /** Mehtod type. */ + typedef void (ValueType::*MethodType)(); + + /** + * Constructor. + * + * @param val Instance, to call method on. + * @param method Method to call. + */ + MethodGuard(ValueType* val, MethodType method) : + val(val), + method(method) + { + // No-op. + } + + /** + * Destructor. + */ + ~MethodGuard() + { + if (val && method) + (val->*method)(); + } + + /** + * Release control over object. + */ + void Release() + { + val = 0; + method = 0; + } + + private: + /** Instance, to call method on. */ + ValueType* val; + + /** Method to call. */ + MethodType method; + }; + /** * Get dynamic library full name. * @param name Name without extension. diff --git a/modules/platforms/cpp/odbc/Makefile.am b/modules/platforms/cpp/odbc/Makefile.am index 5a8ed6decf943..5230a2c5d5993 100644 --- a/modules/platforms/cpp/odbc/Makefile.am +++ b/modules/platforms/cpp/odbc/Makefile.am @@ -57,7 +57,7 @@ libignite_odbc_la_SOURCES = \ src/config/config_tools.cpp \ src/config/configuration.cpp \ src/config/connection_info.cpp \ - src/config/connection_string_parser.cpp \ + src/config/connection_string_parser.cpp \ src/connection.cpp \ src/cursor.cpp \ src/diagnostic/diagnosable_adapter.cpp \ @@ -81,6 +81,7 @@ libignite_odbc_la_SOURCES = \ src/ssl/ssl_gateway.cpp \ src/ssl/secure_socket_client.cpp \ src/ssl/ssl_mode.cpp \ + src/ssl/ssl_api.cpp \ src/sql/sql_lexer.cpp \ src/sql/sql_parser.cpp \ src/sql/sql_set_streaming_command.cpp \ diff --git a/modules/platforms/cpp/odbc/include/Makefile.am b/modules/platforms/cpp/odbc/include/Makefile.am index be6e059edfcfc..a3f6995512b12 100644 --- a/modules/platforms/cpp/odbc/include/Makefile.am +++ b/modules/platforms/cpp/odbc/include/Makefile.am @@ -53,7 +53,7 @@ noinst_HEADERS = \ ignite/odbc/diagnostic/diagnosable.h \ ignite/odbc/diagnostic/diagnosable_adapter.h \ ignite/odbc/ssl/ssl_mode.h \ - ignite/odbc/ssl/ssl_bindings.h \ + ignite/odbc/ssl/ssl_api.h \ ignite/odbc/ssl/secure_socket_client.h \ ignite/odbc/ssl/ssl_gateway.h \ ignite/odbc/sql/sql_command.h \ diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_api.h b/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_api.h new file mode 100644 index 0000000000000..69c7ab540d04d --- /dev/null +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_api.h @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _IGNITE_ODBC_SSL_SSL_API +#define _IGNITE_ODBC_SSL_SSL_API + +namespace ignite +{ + namespace odbc + { + namespace ssl + { + bool EnsureSslLoaded(); + } + } +} + +#endif //_IGNITE_ODBC_SSL_SSL_API \ No newline at end of file diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_bindings.h b/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_bindings.h deleted file mode 100644 index 9a1740dc41f7c..0000000000000 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_bindings.h +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _IGNITE_ODBC_SSL_SSL_BINDINGS -#define _IGNITE_ODBC_SSL_SSL_BINDINGS - -#include -#include -#include - -#include "ignite/odbc/ssl/ssl_gateway.h" - -namespace ignite -{ - namespace odbc - { - namespace ssl - { - // Declaring constant used by OpenSSL for readability. - enum { OPERATION_SUCCESS = 1 }; - - inline SSL_CTX *SSL_CTX_new(const SSL_METHOD *meth) - { - typedef SSL_CTX*(FuncType)(const SSL_METHOD*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_CTX_new); - - return fp(meth); - } - - inline void SSL_CTX_free(SSL_CTX *ctx) - { - typedef void(FuncType)(SSL_CTX*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_CTX_free); - - fp(ctx); - } - - inline void SSL_CTX_set_verify(SSL_CTX *ctx, int mode, int(*callback) (int, X509_STORE_CTX *)) - { - typedef void(FuncType)(SSL_CTX*, int, int(*)(int, X509_STORE_CTX*)); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_set_verify); - - fp(ctx, mode, callback); - } - - inline void SSL_CTX_set_verify_depth(SSL_CTX *ctx, int depth) - { - typedef void(FuncType)(SSL_CTX*, int); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_set_verify_depth); - - fp(ctx, depth); - } - - inline int SSL_CTX_load_verify_locations(SSL_CTX *ctx, const char *cAfile, const char *cApath) - { - typedef int(FuncType)(SSL_CTX*, const char*, const char*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_load_verify_locations); - - return fp(ctx, cAfile, cApath); - } - - inline int SSL_CTX_use_certificate_chain_file(SSL_CTX *ctx, const char *file) - { - typedef int(FuncType)(SSL_CTX*, const char*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_use_certificate_chain_file); - - return fp(ctx, file); - } - - inline int SSL_CTX_use_RSAPrivateKey_file(SSL_CTX *ctx, const char *file, int type) - { - typedef int(FuncType)(SSL_CTX*, const char*, int); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_use_RSAPrivateKey_file); - - return fp(ctx, file, type); - } - - inline int SSL_CTX_set_cipher_list(SSL_CTX *ctx, const char *str) - { - typedef int(FuncType)(SSL_CTX*, const char*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_set_cipher_list); - - return fp(ctx, str); - } - - inline long SSL_CTX_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) - { - typedef long(FuncType)(SSL_CTX*, int, long, void*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_CTX_ctrl); - - return fp(ctx, cmd, larg, parg); - } - - inline long SSL_get_verify_result(const SSL *s) - { - typedef long(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_get_verify_result); - - return fp(s); - } - - inline int SSL_library_init() - { - typedef int(FuncType)(); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_library_init); - - return fp(); - } - - inline void SSL_load_error_strings() - { - typedef void(FuncType)(); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_load_error_strings); - - fp(); - } - - inline X509 *SSL_get_peer_certificate(const SSL *s) - { - typedef X509*(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_get_peer_certificate); - - return fp(s); - } - - inline long SSL_ctrl(SSL *s, int cmd, long larg, void *parg) - { - typedef long(FuncType)(SSL*, int, long, void*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_ctrl); - - return fp(s, cmd, larg ,parg); - } - - inline long SSL_set_tlsext_host_name_(SSL *s, const char *name) - { - return ssl::SSL_ctrl(s, SSL_CTRL_SET_TLSEXT_HOSTNAME, - TLSEXT_NAMETYPE_host_name, const_cast(name)); - } - - inline void SSL_set_connect_state_(SSL* s) - { - typedef void(FuncType)(SSL*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_set_connect_state); - - return fp(s); - } - - inline int SSL_connect_(SSL* s) - { - typedef int(FuncType)(SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_connect); - - return fp(s); - } - - inline int SSL_get_error_(const SSL *s, int ret) - { - typedef int(FuncType)(const SSL*, int); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_get_error); - - return fp(s, ret); - } - - inline int SSL_want_(const SSL *s) - { - typedef int(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_want); - - return fp(s); - } - - inline int SSL_write_(SSL *s, const void *buf, int num) - { - typedef int(FuncType)(SSL*, const void*, int); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_write); - - return fp(s, buf, num); - } - - inline int SSL_read_(SSL *s, void *buf, int num) - { - typedef int(FuncType)(SSL*, void*, int); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_read); - - return fp(s, buf, num); - } - - inline int SSL_pending_(const SSL *ssl) - { - typedef int(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_pending); - - return fp(ssl); - } - - inline int SSL_get_fd_(const SSL *ssl) - { - typedef int(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_get_fd); - - return fp(ssl); - } - - inline void SSL_free_(SSL *ssl) - { - typedef void(FuncType)(SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_free); - - fp(ssl); - } - - inline const SSL_METHOD *SSLv23_client_method_() - { - typedef const SSL_METHOD*(FuncType)(); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSLv23_client_method); - - return fp(); - } - - inline void OPENSSL_config(const char *configName) - { - typedef void(FuncType)(const char*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpOPENSSL_config); - - fp(configName); - } - - inline void X509_free(X509 *a) - { - typedef void(FuncType)(X509*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpX509_free); - - fp(a); - } - - inline BIO *BIO_new_ssl_connect(SSL_CTX *ctx) - { - typedef BIO*(FuncType)(SSL_CTX*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpBIO_new_ssl_connect); - - return fp(ctx); - } - - inline void BIO_free_all(BIO *a) - { - typedef void(FuncType)(BIO*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpBIO_free_all); - - fp(a); - } - - inline long BIO_ctrl(BIO *bp, int cmd, long larg, void *parg) - { - typedef long(FuncType)(BIO*, int, long, void*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpBIO_ctrl); - - return fp(bp, cmd, larg, parg); - } - - inline long BIO_get_fd_(BIO *bp, int *fd) - { - return ssl::BIO_ctrl(bp, BIO_C_GET_FD, 0, reinterpret_cast(fd)); - } - - inline long BIO_get_ssl_(BIO *bp, SSL** ssl) - { - return ssl::BIO_ctrl(bp, BIO_C_GET_SSL, 0, reinterpret_cast(ssl)); - } - - inline long BIO_set_nbio_(BIO *bp, long n) - { - return ssl::BIO_ctrl(bp, BIO_C_SET_NBIO, n, NULL); - } - - inline long BIO_set_conn_hostname_(BIO *bp, const char *name) - { - return ssl::BIO_ctrl(bp, BIO_C_SET_CONNECT, 0, const_cast(name)); - } - - inline unsigned long ERR_get_error_() - { - typedef unsigned long(FuncType)(); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpERR_get_error); - - return fp(); - } - - inline void ERR_error_string_n_(unsigned long e, char *buf, size_t len) - { - typedef void(FuncType)(unsigned long, char*, size_t); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpERR_error_string_n); - - fp(e, buf, len); - } - } - } -} - -#endif //_IGNITE_ODBC_SSL_SSL_BINDINGS \ No newline at end of file diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_gateway.h b/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_gateway.h index 4b102ad28b972..02761b2dc5f32 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_gateway.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/ssl/ssl_gateway.h @@ -18,6 +18,10 @@ #ifndef _IGNITE_ODBC_SSL_SSL_LIBRARY #define _IGNITE_ODBC_SSL_SSL_LIBRARY +#include +#include +#include + #include "ignite/common/concurrent.h" #include "ignite/common/dynamic_load_os.h" @@ -32,6 +36,7 @@ namespace ignite */ struct SslFunctions { + void *fpSSLeay_version; void *fpSSL_CTX_new; void *fpSSL_CTX_free; void *fpSSL_CTX_set_verify; @@ -63,6 +68,12 @@ namespace ignite void *fpBIO_ctrl; void *fpERR_get_error; void *fpERR_error_string_n; + void *fpERR_print_errors_fp; + + void *fpOpenSSL_version; + void *fpSSL_CTX_set_options; + void *fpOPENSSL_init_ssl; + void *fpTLS_client_method; }; /** @@ -103,6 +114,88 @@ namespace ignite return inited; } + char* SSLeay_version_(int type); + + int OPENSSL_init_ssl_(uint64_t opts, const void* settings); + + long SSL_CTX_set_options_(SSL_CTX* ctx, long options); + + long SSL_CTX_ctrl_(SSL_CTX* ctx, int cmd, long larg, void* parg); + + SSL_CTX* SSL_CTX_new_(const SSL_METHOD* meth); + + void SSL_CTX_free_(SSL_CTX* ctx); + + void SSL_CTX_set_verify_(SSL_CTX* ctx, int mode, int (*callback)(int, X509_STORE_CTX*)); + + void SSL_CTX_set_verify_depth_(SSL_CTX* ctx, int depth); + + int SSL_CTX_load_verify_locations_(SSL_CTX* ctx, const char* cAfile, const char* cApath); + + int SSL_CTX_use_certificate_chain_file_(SSL_CTX* ctx, const char* file); + + int SSL_CTX_use_RSAPrivateKey_file_(SSL_CTX* ctx, const char* file, int type); + + int SSL_CTX_set_cipher_list_(SSL_CTX* ctx, const char* str); + + long SSL_get_verify_result_(const SSL* s); + + int SSL_library_init_(); + + void SSL_load_error_strings_(); + + X509* SSL_get_peer_certificate_(const SSL* s); + + long SSL_ctrl_(SSL* s, int cmd, long larg, void* parg); + + long SSL_set_tlsext_host_name_(SSL* s, const char* name); + + void SSL_set_connect_state_(SSL* s); + + int SSL_connect_(SSL* s); + + int SSL_get_error_(const SSL* s, int ret); + + int SSL_want_(const SSL* s); + + int SSL_write_(SSL* s, const void* buf, int num); + + int SSL_read_(SSL* s, void* buf, int num); + + int SSL_pending_(const SSL* ssl); + + int SSL_get_fd_(const SSL* ssl); + + void SSL_free_(SSL* ssl); + + const SSL_METHOD* SSLv23_client_method_(); + + const SSL_METHOD* TLS_client_method_(); + + void OPENSSL_config_(const char* configName); + + void X509_free_(X509* a); + + BIO* BIO_new_ssl_connect_(SSL_CTX* ctx); + + void BIO_free_all_(BIO* a); + + long BIO_ctrl_(BIO* bp, int cmd, long larg, void* parg); + + long BIO_get_fd_(BIO* bp, int* fd); + + long BIO_get_ssl_(BIO* bp, SSL** ssl); + + long BIO_set_nbio_(BIO* bp, long n); + + long BIO_set_conn_hostname_(BIO* bp, const char* name); + + unsigned long ERR_get_error_(); + + void ERR_error_string_n_(unsigned long e, char* buf, size_t len); + + void ERR_print_errors_fp_(FILE *fd); + private: /** * Constructor. @@ -114,6 +207,11 @@ namespace ignite */ ~SslGateway(); + /** + * Unload all SSL symbols. + */ + void UnloadAll(); + /** * Load SSL library. * @param name Name. @@ -126,9 +224,23 @@ namespace ignite */ bool LoadSslLibraries(); + /** + * Load mandatory SSL methods. + * + * @throw IgniteError if can not load one of the functions. + */ + void LoadMandatoryMethods(); + + /** + * Try load SSL method. + * + * @param name Name. + * @return Method pointer. + */ + void* TryLoadSslMethod(const char* name); + /** * Load SSL method. - * @param mod Module. * @param name Name. * @return Method pointer. */ @@ -146,6 +258,9 @@ namespace ignite /** ssleay32 module. */ common::dynamic::Module ssleay32; + /** libcrypto module. */ + common::dynamic::Module libcrypto; + /** libssl module. */ common::dynamic::Module libssl; diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj index 69318ef1e496a..b583c111031df 100644 --- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj +++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj @@ -201,6 +201,7 @@ + @@ -263,7 +264,7 @@ - + diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters index f0e49b576e4c1..5fe3bd367e54f 100644 --- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters +++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters @@ -196,6 +196,9 @@ Code\streaming + + Code\ssl + @@ -344,9 +347,6 @@ Code\ssl - - Code\ssl - Code\config @@ -392,5 +392,8 @@ Code\streaming + + Code\ssl + \ No newline at end of file diff --git a/modules/platforms/cpp/odbc/src/connection.cpp b/modules/platforms/cpp/odbc/src/connection.cpp index b580f12804fea..1953a162620da 100644 --- a/modules/platforms/cpp/odbc/src/connection.cpp +++ b/modules/platforms/cpp/odbc/src/connection.cpp @@ -30,7 +30,7 @@ #include "ignite/odbc/connection.h" #include "ignite/odbc/message.h" #include "ignite/odbc/ssl/ssl_mode.h" -#include "ignite/odbc/ssl/ssl_gateway.h" +#include "ignite/odbc/ssl/ssl_api.h" #include "ignite/odbc/ssl/secure_socket_client.h" #include "ignite/odbc/system/tcp_socket_client.h" #include "ignite/odbc/dsn_config.h" @@ -155,7 +155,7 @@ namespace ignite if (sslMode != SslMode::DISABLE) { - bool loaded = ssl::SslGateway::GetInstance().LoadAll(); + bool loaded = ssl::EnsureSslLoaded(); if (!loaded) { diff --git a/modules/platforms/cpp/odbc/src/ssl/secure_socket_client.cpp b/modules/platforms/cpp/odbc/src/ssl/secure_socket_client.cpp index 84eb1e8867f82..f7811fb7925b6 100644 --- a/modules/platforms/cpp/odbc/src/ssl/secure_socket_client.cpp +++ b/modules/platforms/cpp/odbc/src/ssl/secure_socket_client.cpp @@ -22,13 +22,15 @@ #include "ignite/common/concurrent.h" #include "ignite/odbc/system/tcp_socket_client.h" #include "ignite/odbc/ssl/secure_socket_client.h" -#include "ignite/odbc/ssl/ssl_bindings.h" +#include "ignite/odbc/ssl/ssl_gateway.h" #include "ignite/common/utils.h" #ifndef SOCKET_ERROR # define SOCKET_ERROR (-1) #endif // SOCKET_ERROR +enum { OPERATION_SUCCESS = 1 }; + namespace ignite { namespace odbc @@ -52,13 +54,15 @@ namespace ignite CloseInteral(); if (context) - ssl::SSL_CTX_free(reinterpret_cast(context)); + SslGateway::GetInstance().SSL_CTX_free_(reinterpret_cast(context)); } bool SecureSocketClient::Connect(const char* hostname, uint16_t port, int32_t, diagnostic::Diagnosable& diag) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (!context) { @@ -77,52 +81,52 @@ namespace ignite if (!ssl0) return false; - int res = ssl::SSL_set_tlsext_host_name_(ssl0, hostname); + int res = sslGateway.SSL_set_tlsext_host_name_(ssl0, hostname); if (res != OPERATION_SUCCESS) { + sslGateway.SSL_free_(ssl0); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set host name for secure connection: " + GetSslError(ssl0, res)); - ssl::SSL_free_(ssl0); - return false; } - ssl::SSL_set_connect_state_(ssl0); + sslGateway.SSL_set_connect_state_(ssl0); bool connected = CompleteConnectInternal(ssl0, DEFALT_CONNECT_TIMEOUT, diag); if (!connected) { - ssl::SSL_free_(ssl0); + sslGateway.SSL_free_(ssl0); return false; } // Verify a server certificate was presented during the negotiation - X509* cert = ssl::SSL_get_peer_certificate(ssl0); + X509* cert = sslGateway.SSL_get_peer_certificate_(ssl0); if (cert) - ssl::X509_free(cert); + sslGateway.X509_free_(cert); else { + sslGateway.SSL_free_(ssl0); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Remote host did not provide certificate: " + GetSslError(ssl0, res)); - ssl::SSL_free_(ssl0); - return false; } // Verify the result of chain verification // Verification performed according to RFC 4158 - res = ssl::SSL_get_verify_result(ssl0); + res = sslGateway.SSL_get_verify_result_(ssl0); if (X509_V_OK != res) { + sslGateway.SSL_free_(ssl0); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Certificate chain verification failed: " + GetSslError(ssl0, res)); - ssl::SSL_free_(ssl0); - return false; } @@ -138,7 +142,9 @@ namespace ignite int SecureSocketClient::Send(const int8_t* data, size_t size, int32_t timeout) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (!ssl) { @@ -149,14 +155,16 @@ namespace ignite SSL* ssl0 = reinterpret_cast(ssl); - int res = ssl::SSL_write_(ssl0, data, static_cast(size)); + int res = sslGateway.SSL_write_(ssl0, data, static_cast(size)); return res; } int SecureSocketClient::Receive(int8_t* buffer, size_t size, int32_t timeout) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (!ssl) { @@ -169,7 +177,7 @@ namespace ignite int res = 0; - if (!blocking && ssl::SSL_pending_(ssl0) == 0) + if (!blocking && sslGateway.SSL_pending_(ssl0) == 0) { res = WaitOnSocket(ssl, timeout, true); @@ -177,7 +185,7 @@ namespace ignite return res; } - res = ssl::SSL_read_(ssl0, buffer, static_cast(size)); + res = sslGateway.SSL_read_(ssl0, buffer, static_cast(size)); return res; } @@ -190,30 +198,11 @@ namespace ignite void* SecureSocketClient::MakeContext(const std::string& certPath, const std::string& keyPath, const std::string& caPath, diagnostic::Diagnosable& diag) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); - static bool sslLibInited = false; - static common::concurrent::CriticalSection sslCs; + assert(sslGateway.Loaded()); - if (!sslLibInited) - { - common::concurrent::CsLockGuard lock(sslCs); - - if (!sslLibInited) - { - LOG_MSG("Initializing SSL library"); - - (void)SSL_library_init(); - - SSL_load_error_strings(); - - OPENSSL_config(0); - - sslLibInited = true; - } - } - - const SSL_METHOD* method = ssl::SSLv23_client_method_(); + const SSL_METHOD* method = sslGateway.SSLv23_client_method_(); if (!method) { diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not get SSL method."); @@ -221,7 +210,7 @@ namespace ignite return 0; } - SSL_CTX* ctx = ssl::SSL_CTX_new(method); + SSL_CTX* ctx = sslGateway.SSL_CTX_new_(method); if (!ctx) { diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not create new SSL context."); @@ -229,57 +218,56 @@ namespace ignite return 0; } - ssl::SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, 0); + sslGateway.SSL_CTX_set_verify_(ctx, SSL_VERIFY_PEER, 0); - ssl::SSL_CTX_set_verify_depth(ctx, 8); + sslGateway.SSL_CTX_set_verify_depth_(ctx, 8); - const long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION; - ssl::SSL_CTX_ctrl(ctx, SSL_CTRL_OPTIONS, flags, NULL); + sslGateway.SSL_CTX_set_options_(ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION); const char* cCaPath = caPath.empty() ? 0 : caPath.c_str(); - long res = ssl::SSL_CTX_load_verify_locations(ctx, cCaPath, 0); + long res = sslGateway.SSL_CTX_load_verify_locations_(ctx, cCaPath, 0); if (res != OPERATION_SUCCESS) { + sslGateway.SSL_CTX_free_(ctx); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set Certificate Authority path for secure connection."); - ssl::SSL_CTX_free(ctx); - return 0; } - res = ssl::SSL_CTX_use_certificate_chain_file(ctx, certPath.c_str()); + res = sslGateway.SSL_CTX_use_certificate_chain_file_(ctx, certPath.c_str()); if (res != OPERATION_SUCCESS) { + sslGateway.SSL_CTX_free_(ctx); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set client certificate file for secure connection."); - ssl::SSL_CTX_free(ctx); - return 0; } - res = ssl::SSL_CTX_use_RSAPrivateKey_file(ctx, keyPath.c_str(), SSL_FILETYPE_PEM); + res = sslGateway.SSL_CTX_use_RSAPrivateKey_file_(ctx, keyPath.c_str(), SSL_FILETYPE_PEM); if (res != OPERATION_SUCCESS) { + sslGateway.SSL_CTX_free_(ctx); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set private key file for secure connection."); - ssl::SSL_CTX_free(ctx); - return 0; } const char* const PREFERRED_CIPHERS = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4"; - res = ssl::SSL_CTX_set_cipher_list(ctx, PREFERRED_CIPHERS); + res = sslGateway.SSL_CTX_set_cipher_list_(ctx, PREFERRED_CIPHERS); if (res != OPERATION_SUCCESS) { + sslGateway.SSL_CTX_free_(ctx); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set ciphers list for secure connection."); - ssl::SSL_CTX_free(ctx); - return 0; } @@ -289,7 +277,11 @@ namespace ignite void* SecureSocketClient::MakeSsl(void* context, const char* hostname, uint16_t port, bool& blocking, diagnostic::Diagnosable& diag) { - BIO* bio = ssl::BIO_new_ssl_connect(reinterpret_cast(context)); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + + BIO* bio = sslGateway.BIO_new_ssl_connect_(reinterpret_cast(context)); if (!bio) { diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not create SSL connection."); @@ -298,7 +290,7 @@ namespace ignite } blocking = false; - long res = ssl::BIO_set_nbio_(bio, 1); + long res = sslGateway.BIO_set_nbio_(bio, 1); if (res != OPERATION_SUCCESS) { blocking = true; @@ -312,23 +304,23 @@ namespace ignite std::string address = stream.str(); - res = ssl::BIO_set_conn_hostname_(bio, address.c_str()); + res = sslGateway.BIO_set_conn_hostname_(bio, address.c_str()); if (res != OPERATION_SUCCESS) { - diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set SSL connection hostname."); + sslGateway.BIO_free_all_(bio); - ssl::BIO_free_all(bio); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not set SSL connection hostname."); return 0; } SSL* ssl = 0; - ssl::BIO_get_ssl_(bio, &ssl); + sslGateway.BIO_get_ssl_(bio, &ssl); if (!ssl) { - diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not get SSL instance from BIO."); + sslGateway.BIO_free_all_(bio); - ssl::BIO_free_all(bio); + diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, "Can not get SSL instance from BIO."); return 0; } @@ -338,16 +330,20 @@ namespace ignite bool SecureSocketClient::CompleteConnectInternal(void* ssl, int timeout, diagnostic::Diagnosable& diag) { + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + SSL* ssl0 = reinterpret_cast(ssl); while (true) { - int res = ssl::SSL_connect_(ssl0); + int res = sslGateway.SSL_connect_(ssl0); if (res == OPERATION_SUCCESS) return true; - int sslError = ssl::SSL_get_error_(ssl0, res); + int sslError = sslGateway.SSL_get_error_(ssl0, res); LOG_MSG("wait res=" << res << ", sslError=" << sslError); @@ -359,7 +355,7 @@ namespace ignite return false; } - int want = ssl::SSL_want_(ssl0); + int want = sslGateway.SSL_want_(ssl0); res = WaitOnSocket(ssl, timeout, want == SSL_READING); @@ -386,9 +382,13 @@ namespace ignite std::string SecureSocketClient::GetSslError(void* ssl, int ret) { + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + SSL* ssl0 = reinterpret_cast(ssl); - int sslError = ssl::SSL_get_error_(ssl0, ret); + int sslError = sslGateway.SSL_get_error_(ssl0, ret); LOG_MSG("ssl_error: " << sslError); @@ -407,11 +407,11 @@ namespace ignite return std::string("SSL error: ") + common::LexicalCast(sslError); } - long error = ssl::ERR_get_error_(); + long error = sslGateway.ERR_get_error_(); char errBuf[1024] = { 0 }; - ssl::ERR_error_string_n_(error, errBuf, sizeof(errBuf)); + sslGateway.ERR_error_string_n_(error, errBuf, sizeof(errBuf)); return std::string(errBuf); } @@ -435,11 +435,13 @@ namespace ignite void SecureSocketClient::CloseInteral() { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (ssl) { - ssl::SSL_free_(reinterpret_cast(ssl)); + sslGateway.SSL_free_(reinterpret_cast(ssl)); ssl = 0; } @@ -447,13 +449,17 @@ namespace ignite int SecureSocketClient::WaitOnSocket(void* ssl, int32_t timeout, bool rd) { + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + int ready = 0; int lastError = 0; SSL* ssl0 = reinterpret_cast(ssl); fd_set fds; - int fd = ssl::SSL_get_fd_(ssl0); + int fd = sslGateway.SSL_get_fd_(ssl0); if (fd < 0) { diff --git a/modules/platforms/cpp/odbc/src/ssl/ssl_api.cpp b/modules/platforms/cpp/odbc/src/ssl/ssl_api.cpp new file mode 100644 index 0000000000000..4a1d6926c2765 --- /dev/null +++ b/modules/platforms/cpp/odbc/src/ssl/ssl_api.cpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ignite/odbc/ssl/ssl_gateway.h" +#include "ignite/odbc/ssl/ssl_api.h" + +namespace ignite +{ + namespace odbc + { + namespace ssl + { + bool EnsureSslLoaded() + { + return SslGateway::GetInstance().LoadAll(); + } + } + } +} diff --git a/modules/platforms/cpp/odbc/src/ssl/ssl_gateway.cpp b/modules/platforms/cpp/odbc/src/ssl/ssl_gateway.cpp index 308302a069f8a..6116a767a7e69 100644 --- a/modules/platforms/cpp/odbc/src/ssl/ssl_gateway.cpp +++ b/modules/platforms/cpp/odbc/src/ssl/ssl_gateway.cpp @@ -17,9 +17,6 @@ #include -#include -#include - #include "ignite/common/utils.h" #include "ignite/odbc/ssl/ssl_gateway.h" @@ -29,6 +26,18 @@ # define ADDITIONAL_OPENSSL_HOME_ENV "OPEN_SSL_HOME" #endif // ADDITIONAL_OPENSSL_HOME_ENV +#ifndef SSL_CTRL_OPTIONS +# define SSL_CTRL_OPTIONS 32 +#endif // SSL_CTRL_OPTIONS + +#ifndef OPENSSL_INIT_LOAD_SSL_STRINGS +# define OPENSSL_INIT_LOAD_SSL_STRINGS 0x00200000L +#endif // OPENSSL_INIT_LOAD_SSL_STRINGS + +#ifndef OPENSSL_INIT_LOAD_CRYPTO_STRINGS +# define OPENSSL_INIT_LOAD_CRYPTO_STRINGS 0x00000002L +#endif // OPENSSL_INIT_LOAD_CRYPTO_STRINGS + namespace ignite { namespace odbc @@ -39,7 +48,7 @@ namespace ignite inited(false), functions() { - // No-op. + memset(&functions, 0, sizeof(functions)); } SslGateway::~SslGateway() @@ -47,41 +56,61 @@ namespace ignite // No-op. } + void SslGateway::UnloadAll() + { + libeay32.Unload(); + ssleay32.Unload(); + libssl.Unload(); + libcrypto.Unload(); + + memset(&functions, 0, sizeof(functions)); + } + common::dynamic::Module SslGateway::LoadSslLibrary(const char* name) { using namespace common; using namespace dynamic; - std::string fullName = GetDynamicLibraryName(name); - - Module libModule = LoadModule(fullName); - - if (libModule.IsLoaded()) - return libModule; - std::string home = GetEnv(ADDITIONAL_OPENSSL_HOME_ENV); if (home.empty()) home = GetEnv("OPENSSL_HOME"); - if (home.empty()) - return libModule; + std::string fullName = GetDynamicLibraryName(name); + + if (!home.empty()) + { + std::stringstream constructor; + + constructor << home << Fs << "bin" << Fs << fullName; - std::stringstream constructor; + std::string fullPath = constructor.str(); - constructor << home << Fs << "bin" << Fs << fullName; + Module mod = LoadModule(fullPath); - std::string fullPath = constructor.str(); + if (mod.IsLoaded()) + return mod; + } - return LoadModule(fullPath); + return LoadModule(fullName); } bool SslGateway::LoadSslLibraries() { - libeay32 = LoadSslLibrary("libeay32"); - ssleay32 = LoadSslLibrary("ssleay32"); libssl = LoadSslLibrary("libssl"); + if (!libssl.IsLoaded()) + { + libcrypto = LoadSslLibrary("libcrypto-1_1-x64"); + libssl = LoadSslLibrary("libssl-1_1-x64"); + } + + if (!libssl.IsLoaded()) + { + libeay32 = LoadSslLibrary("libeay32"); + ssleay32 = LoadSslLibrary("ssleay32"); + } + if (!libssl.IsLoaded() && (!libeay32.IsLoaded() || !ssleay32.IsLoaded())) { if (!libeay32.IsLoaded()) @@ -93,41 +122,29 @@ namespace ignite if (!libssl.IsLoaded()) LOG_MSG("Can not load libssl."); - libeay32.Unload(); - ssleay32.Unload(); - libssl.Unload(); - return false; } return true; } - SslGateway& SslGateway::GetInstance() - { - static SslGateway self; - - return self; - } - - bool SslGateway::LoadAll() + void SslGateway::LoadMandatoryMethods() { - using namespace common::dynamic; + functions.fpSSLeay_version = TryLoadSslMethod("SSLeay_version"); - if (inited) - return true; + if (!functions.fpSSLeay_version) + functions.fpOpenSSL_version = LoadSslMethod("OpenSSL_version"); - common::concurrent::CsLockGuard lock(initCs); + functions.fpSSL_library_init = TryLoadSslMethod("SSL_library_init"); + functions.fpSSL_load_error_strings = TryLoadSslMethod("SSL_load_error_strings"); - if (inited) - return true; + if (!functions.fpSSL_library_init || !functions.fpSSL_load_error_strings) + functions.fpOPENSSL_init_ssl = LoadSslMethod("OPENSSL_init_ssl"); - if (!LoadSslLibraries()) - { - LOG_MSG("Can not load neccessary OpenSSL libraries."); + functions.fpSSLv23_client_method = TryLoadSslMethod("SSLv23_client_method"); - return false; - } + if (!functions.fpSSLv23_client_method) + functions.fpTLS_client_method = LoadSslMethod("TLS_client_method"); functions.fpSSL_CTX_new = LoadSslMethod("SSL_CTX_new"); functions.fpSSL_CTX_free = LoadSslMethod("SSL_CTX_free"); @@ -139,13 +156,11 @@ namespace ignite functions.fpSSL_CTX_set_cipher_list = LoadSslMethod("SSL_CTX_set_cipher_list"); functions.fpSSL_get_verify_result = LoadSslMethod("SSL_get_verify_result"); - functions.fpSSL_library_init = LoadSslMethod("SSL_library_init"); - functions.fpSSL_load_error_strings = LoadSslMethod("SSL_load_error_strings"); + functions.fpSSL_get_peer_certificate = LoadSslMethod("SSL_get_peer_certificate"); functions.fpSSL_ctrl = LoadSslMethod("SSL_ctrl"); functions.fpSSL_CTX_ctrl = LoadSslMethod("SSL_CTX_ctrl"); - functions.fpSSLv23_client_method = LoadSslMethod("SSLv23_client_method"); functions.fpSSL_set_connect_state = LoadSslMethod("SSL_set_connect_state"); functions.fpSSL_connect = LoadSslMethod("SSL_connect"); functions.fpSSL_get_error = LoadSslMethod("SSL_get_error"); @@ -165,67 +180,521 @@ namespace ignite functions.fpERR_get_error = LoadSslMethod("ERR_get_error"); functions.fpERR_error_string_n = LoadSslMethod("ERR_error_string_n"); + } + + SslGateway& SslGateway::GetInstance() + { + static SslGateway self; + + return self; + } + + bool SslGateway::LoadAll() + { + using namespace common::dynamic; + + if (inited) + return true; + + common::concurrent::CsLockGuard lock(initCs); - bool allLoaded = - functions.fpSSL_CTX_new != 0 && - functions.fpSSL_CTX_free != 0 && - functions.fpSSL_CTX_set_verify != 0 && - functions.fpSSL_CTX_set_verify_depth != 0 && - functions.fpSSL_CTX_load_verify_locations != 0 && - functions.fpSSL_CTX_use_certificate_chain_file != 0 && - functions.fpSSL_CTX_use_RSAPrivateKey_file != 0 && - functions.fpSSL_CTX_set_cipher_list != 0 && - functions.fpSSL_get_verify_result != 0 && - functions.fpSSL_library_init != 0 && - functions.fpSSL_load_error_strings != 0 && - functions.fpSSL_get_peer_certificate != 0 && - functions.fpSSL_ctrl != 0 && - functions.fpSSL_CTX_ctrl != 0 && - functions.fpSSLv23_client_method != 0 && - functions.fpSSL_set_connect_state != 0 && - functions.fpSSL_connect != 0 && - functions.fpSSL_get_error != 0 && - functions.fpSSL_want != 0 && - functions.fpSSL_write != 0 && - functions.fpSSL_read != 0 && - functions.fpSSL_pending != 0 && - functions.fpSSL_get_fd != 0 && - functions.fpSSL_free != 0 && - functions.fpBIO_new_ssl_connect != 0 && - functions.fpOPENSSL_config != 0 && - functions.fpX509_free != 0 && - functions.fpBIO_free_all != 0 && - functions.fpBIO_ctrl != 0 && - functions.fpERR_get_error != 0 && - functions.fpERR_error_string_n != 0; - - if (!allLoaded) + if (inited) + return true; + + common::MethodGuard guard(this, &SslGateway::UnloadAll); + + if (!LoadSslLibraries()) { - libeay32.Unload(); - ssleay32.Unload(); - libssl.Unload(); + LOG_MSG("Can not load neccessary OpenSSL libraries."); + + return false; } - inited = allLoaded; + LoadMandatoryMethods(); + + functions.fpSSL_CTX_set_options = TryLoadSslMethod("SSL_CTX_set_options"); + functions.fpERR_print_errors_fp = TryLoadSslMethod("ERR_print_errors_fp"); + + (void)SSL_library_init_(); - return inited; + SSL_load_error_strings_(); + + OPENSSL_config_(0); + + guard.Release(); + + inited = true; + + return true; } - void* SslGateway::LoadSslMethod(const char* name) + void* SslGateway::TryLoadSslMethod(const char* name) { void* fp = libeay32.FindSymbol(name); if (!fp) fp = ssleay32.FindSymbol(name); + if (!fp) + fp = libcrypto.FindSymbol(name); + if (!fp) fp = libssl.FindSymbol(name); + return fp; + } + + void* SslGateway::LoadSslMethod(const char* name) + { + void* fp = TryLoadSslMethod(name); + if (!fp) LOG_MSG("Can not load function " << name); return fp; } + + char* SslGateway::SSLeay_version_(int type) + { + typedef char* (FuncType)(int); + + FuncType* fp = 0; + + if (functions.fpSSLeay_version) + fp = reinterpret_cast(functions.fpSSLeay_version); + else + fp = reinterpret_cast(functions.fpOpenSSL_version); + + assert(fp != 0); + + return fp(type); + } + + int SslGateway::OPENSSL_init_ssl_(uint64_t opts, const void* settings) + { + assert(functions.fpOPENSSL_init_ssl != 0); + + typedef int (FuncType)(uint64_t, const void*); + + FuncType* fp = reinterpret_cast(functions.fpOPENSSL_init_ssl); + + return fp(opts, settings); + } + + long SslGateway::SSL_CTX_set_options_(SSL_CTX* ctx, long options) + { + if (functions.fpSSL_CTX_set_options) + { + typedef long (FuncType)(SSL_CTX*, long); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_options); + + return fp(ctx, options); + } + + return SSL_CTX_ctrl_(ctx, SSL_CTRL_OPTIONS, options, NULL); + } + + long SslGateway::SSL_CTX_ctrl_(SSL_CTX* ctx, int cmd, long larg, void* parg) + { + assert(functions.fpSSL_CTX_ctrl != 0); + + typedef long (FuncType)(SSL_CTX*, int, long, void*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_ctrl); + + return fp(ctx, cmd, larg, parg); + } + + SSL_CTX* SslGateway::SSL_CTX_new_(const SSL_METHOD* meth) + { + assert(functions.fpSSL_CTX_new != 0); + + typedef SSL_CTX*(FuncType)(const SSL_METHOD*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_new); + + return fp(meth); + } + + void SslGateway::SSL_CTX_free_(SSL_CTX* ctx) + { + assert(functions.fpSSL_CTX_free != 0); + + typedef void (FuncType)(SSL_CTX*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_free); + + fp(ctx); + } + + void SslGateway::SSL_CTX_set_verify_(SSL_CTX* ctx, int mode, int (* callback)(int, X509_STORE_CTX*)) + { + assert(functions.fpSSL_CTX_set_verify != 0); + + typedef void (FuncType)(SSL_CTX*, int, int (*)(int, X509_STORE_CTX*)); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_verify); + + fp(ctx, mode, callback); + } + + void SslGateway::SSL_CTX_set_verify_depth_(SSL_CTX* ctx, int depth) + { + assert(functions.fpSSL_CTX_set_verify_depth != 0); + + typedef void (FuncType)(SSL_CTX*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_verify_depth); + + fp(ctx, depth); + } + + int SslGateway::SSL_CTX_load_verify_locations_(SSL_CTX* ctx, const char* cAfile, const char* cApath) + { + assert(functions.fpSSL_CTX_load_verify_locations != 0); + + typedef int (FuncType)(SSL_CTX*, const char*, const char*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_load_verify_locations); + + return fp(ctx, cAfile, cApath); + } + + int SslGateway::SSL_CTX_use_certificate_chain_file_(SSL_CTX* ctx, const char* file) + { + assert(functions.fpSSL_CTX_use_certificate_chain_file != 0); + + typedef int (FuncType)(SSL_CTX*, const char*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_use_certificate_chain_file); + + return fp(ctx, file); + } + + int SslGateway::SSL_CTX_use_RSAPrivateKey_file_(SSL_CTX* ctx, const char* file, int type) + { + assert(functions.fpSSL_CTX_use_RSAPrivateKey_file != 0); + + typedef int (FuncType)(SSL_CTX*, const char*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_use_RSAPrivateKey_file); + + return fp(ctx, file, type); + } + + int SslGateway::SSL_CTX_set_cipher_list_(SSL_CTX* ctx, const char* str) + { + assert(functions.fpSSL_CTX_set_cipher_list != 0); + + typedef int (FuncType)(SSL_CTX*, const char*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_cipher_list); + + return fp(ctx, str); + } + + long SslGateway::SSL_get_verify_result_(const SSL* s) + { + assert(functions.fpSSL_get_verify_result != 0); + + typedef long (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_verify_result); + + return fp(s); + } + + int SslGateway::SSL_library_init_() + { + typedef int (FuncType)(); + + if (functions.fpSSL_library_init) + { + FuncType* fp = reinterpret_cast(functions.fpSSL_library_init); + + return fp(); + } + + return OPENSSL_init_ssl_(0, NULL); + } + + void SslGateway::SSL_load_error_strings_() + { + typedef void (FuncType)(); + + if (functions.fpSSL_load_error_strings) + { + FuncType* fp = reinterpret_cast(functions.fpSSL_load_error_strings); + + fp(); + + return; + } + + OPENSSL_init_ssl_(OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); + } + + X509* SslGateway::SSL_get_peer_certificate_(const SSL* s) + { + assert(functions.fpSSL_get_peer_certificate != 0); + + typedef X509*(FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_peer_certificate); + + return fp(s); + } + + long SslGateway::SSL_ctrl_(SSL* s, int cmd, long larg, void* parg) + { + assert(functions.fpSSL_ctrl != 0); + + typedef long (FuncType)(SSL*, int, long, void*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_ctrl); + + return fp(s, cmd, larg, parg); + } + + long SslGateway::SSL_set_tlsext_host_name_(SSL* s, const char* name) + { + return SSL_ctrl_(s, SSL_CTRL_SET_TLSEXT_HOSTNAME, + TLSEXT_NAMETYPE_host_name, const_cast(name)); + } + + void SslGateway::SSL_set_connect_state_(SSL* s) + { + assert(functions.fpSSL_set_connect_state != 0); + + typedef void (FuncType)(SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_set_connect_state); + + return fp(s); + } + + int SslGateway::SSL_connect_(SSL* s) + { + assert(functions.fpSSL_connect != 0); + + typedef int (FuncType)(SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_connect); + + return fp(s); + } + + int SslGateway::SSL_get_error_(const SSL* s, int ret) + { + assert(functions.fpSSL_get_error != 0); + + typedef int (FuncType)(const SSL*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_error); + + return fp(s, ret); + } + + int SslGateway::SSL_want_(const SSL* s) + { + assert(functions.fpSSL_want != 0); + + typedef int (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_want); + + return fp(s); + } + + int SslGateway::SSL_write_(SSL* s, const void* buf, int num) + { + assert(functions.fpSSL_write != 0); + + typedef int (FuncType)(SSL*, const void*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_write); + + return fp(s, buf, num); + } + + int SslGateway::SSL_read_(SSL* s, void* buf, int num) + { + assert(functions.fpSSL_read != 0); + + typedef int (FuncType)(SSL*, void*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_read); + + return fp(s, buf, num); + } + + int SslGateway::SSL_pending_(const SSL* ssl) + { + assert(functions.fpSSL_pending != 0); + + typedef int (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_pending); + + return fp(ssl); + } + + int SslGateway::SSL_get_fd_(const SSL* ssl) + { + assert(functions.fpSSL_get_fd != 0); + + typedef int (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_fd); + + return fp(ssl); + } + + void SslGateway::SSL_free_(SSL* ssl) + { + assert(functions.fpSSL_free != 0); + + typedef void (FuncType)(SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_free); + + fp(ssl); + } + + const SSL_METHOD* SslGateway::SSLv23_client_method_() + { + if (functions.fpSSLv23_client_method) + { + typedef const SSL_METHOD*(FuncType)(); + + FuncType* fp = reinterpret_cast(functions.fpSSLv23_client_method); + + return fp(); + } + + return TLS_client_method_(); + } + + const SSL_METHOD* SslGateway::TLS_client_method_() + { + assert(functions.fpTLS_client_method != 0); + + typedef const SSL_METHOD*(FuncType)(); + + FuncType* fp = reinterpret_cast(functions.fpTLS_client_method); + + return fp(); + } + + void SslGateway::OPENSSL_config_(const char* configName) + { + assert(functions.fpOPENSSL_config != 0); + + typedef void (FuncType)(const char*); + + FuncType* fp = reinterpret_cast(functions.fpOPENSSL_config); + + fp(configName); + } + + void SslGateway::X509_free_(X509* a) + { + assert(functions.fpX509_free != 0); + + typedef void (FuncType)(X509*); + + FuncType* fp = reinterpret_cast(functions.fpX509_free); + + fp(a); + } + + BIO* SslGateway::BIO_new_ssl_connect_(SSL_CTX* ctx) + { + assert(functions.fpBIO_new_ssl_connect != 0); + + typedef BIO*(FuncType)(SSL_CTX*); + + FuncType* fp = reinterpret_cast(functions.fpBIO_new_ssl_connect); + + return fp(ctx); + } + + void SslGateway::BIO_free_all_(BIO* a) + { + assert(functions.fpBIO_free_all != 0); + + typedef void (FuncType)(BIO*); + + FuncType* fp = reinterpret_cast(functions.fpBIO_free_all); + + fp(a); + } + + long SslGateway::BIO_ctrl_(BIO* bp, int cmd, long larg, void* parg) + { + assert(functions.fpBIO_ctrl != 0); + + typedef long (FuncType)(BIO*, int, long, void*); + + FuncType* fp = reinterpret_cast(functions.fpBIO_ctrl); + + return fp(bp, cmd, larg, parg); + } + + long SslGateway::BIO_get_fd_(BIO* bp, int* fd) + { + return BIO_ctrl_(bp, BIO_C_GET_FD, 0, reinterpret_cast(fd)); + } + + long SslGateway::BIO_get_ssl_(BIO* bp, SSL** ssl) + { + return BIO_ctrl_(bp, BIO_C_GET_SSL, 0, reinterpret_cast(ssl)); + } + + long SslGateway::BIO_set_nbio_(BIO* bp, long n) + { + return BIO_ctrl_(bp, BIO_C_SET_NBIO, n, NULL); + } + + long SslGateway::BIO_set_conn_hostname_(BIO* bp, const char* name) + { + return BIO_ctrl_(bp, BIO_C_SET_CONNECT, 0, const_cast(name)); + } + + unsigned long SslGateway::ERR_get_error_() + { + assert(functions.fpERR_get_error != 0); + + typedef unsigned long (FuncType)(); + + FuncType* fp = reinterpret_cast(functions.fpERR_get_error); + + return fp(); + } + + void SslGateway::ERR_error_string_n_(unsigned long e, char* buf, size_t len) + { + assert(functions.fpERR_error_string_n != 0); + + typedef void (FuncType)(unsigned long, char*, size_t); + + FuncType* fp = reinterpret_cast(functions.fpERR_error_string_n); + + fp(e, buf, len); + } + + void SslGateway::ERR_print_errors_fp_(FILE* fd) + { + if (!functions.fpERR_print_errors_fp) + return; + + typedef void (FuncType)(FILE*); + + FuncType* fp = reinterpret_cast(functions.fpERR_print_errors_fp); + + fp(fd); + } } } } diff --git a/modules/platforms/cpp/thin-client-test/src/ssl_test.cpp b/modules/platforms/cpp/thin-client-test/src/ssl_test.cpp index 6bfda493a3005..eaadd9b26c59f 100644 --- a/modules/platforms/cpp/thin-client-test/src/ssl_test.cpp +++ b/modules/platforms/cpp/thin-client-test/src/ssl_test.cpp @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(SslConnectionReject2) cfg.SetEndPoints("127.0.0.1:11110"); cfg.SetSslMode(SslMode::DISABLE); - + BOOST_CHECK_THROW(IgniteClient::Start(cfg), ignite::IgniteError); } diff --git a/modules/platforms/cpp/thin-client/Makefile.am b/modules/platforms/cpp/thin-client/Makefile.am index f8d919012fade..1efff8ea44341 100644 --- a/modules/platforms/cpp/thin-client/Makefile.am +++ b/modules/platforms/cpp/thin-client/Makefile.am @@ -63,6 +63,7 @@ libignite_thin_client_la_SOURCES = \ src/impl/data_router.cpp \ src/impl/ssl/ssl_gateway.cpp \ src/impl/ssl/secure_socket_client.cpp \ + src/impl/ssl/ssl_api.cpp \ src/ignite_client.cpp clean-local: diff --git a/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj b/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj index 3221268d37f1f..9c0091380cafa 100644 --- a/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj +++ b/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj @@ -166,6 +166,7 @@ + @@ -195,7 +196,7 @@ - + @@ -210,4 +211,4 @@ - + \ No newline at end of file diff --git a/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj.filters b/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj.filters index 2d20106f35572..a5cad74bcb705 100644 --- a/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj.filters +++ b/modules/platforms/cpp/thin-client/project/vs/thin-client.vcxproj.filters @@ -67,6 +67,9 @@ Code\impl\cache + + Code\impl\ssl + @@ -123,9 +126,6 @@ Code\impl\ssl - - Code\impl\ssl - Code\impl\ssl @@ -153,5 +153,8 @@ Code\impl\cache + + Code\impl\ssl + - + \ No newline at end of file diff --git a/modules/platforms/cpp/thin-client/src/impl/data_channel.cpp b/modules/platforms/cpp/thin-client/src/impl/data_channel.cpp index 6b40d811b64f0..592dba331fc9e 100644 --- a/modules/platforms/cpp/thin-client/src/impl/data_channel.cpp +++ b/modules/platforms/cpp/thin-client/src/impl/data_channel.cpp @@ -23,7 +23,7 @@ #include #include "impl/message.h" -#include "impl/ssl/ssl_gateway.h" +#include "impl/ssl/ssl_api.h" #include "impl/ssl/secure_socket_client.h" #include "impl/net/tcp_socket_client.h" #include "impl/net/remote_type_updater.h" @@ -81,7 +81,7 @@ namespace ignite if (sslMode != SslMode::DISABLE) { - ssl::SslGateway::GetInstance().LoadAll(); + ssl::EnsureSslLoaded(); socket.reset(new ssl::SecureSocketClient(config.GetSslCertFile(), config.GetSslKeyFile(), config.GetSslCaFile())); diff --git a/modules/platforms/cpp/thin-client/src/impl/ssl/secure_socket_client.cpp b/modules/platforms/cpp/thin-client/src/impl/ssl/secure_socket_client.cpp index e7e60b6e55e61..5a9b7f4afd24c 100644 --- a/modules/platforms/cpp/thin-client/src/impl/ssl/secure_socket_client.cpp +++ b/modules/platforms/cpp/thin-client/src/impl/ssl/secure_socket_client.cpp @@ -24,12 +24,15 @@ #include "impl/net/tcp_socket_client.h" #include "impl/ssl/secure_socket_client.h" -#include "impl/ssl/ssl_bindings.h" + +#include "impl/ssl/ssl_gateway.h" #ifndef SOCKET_ERROR # define SOCKET_ERROR (-1) #endif // SOCKET_ERROR +enum { OPERATION_SUCCESS = 1 }; + namespace ignite { namespace impl @@ -55,12 +58,14 @@ namespace ignite CloseInteral(); if (context) - ssl::SSL_CTX_free(reinterpret_cast(context)); + SslGateway::GetInstance().SSL_CTX_free_(reinterpret_cast(context)); } bool SecureSocketClient::Connect(const char* hostname, uint16_t port, int32_t timeout) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (!context) { @@ -77,34 +82,34 @@ namespace ignite if (!ssl0) return false; - int res = ssl::SSL_set_tlsext_host_name_(ssl0, hostname); + int res = sslGateway.SSL_set_tlsext_host_name_(ssl0, hostname); if (res != OPERATION_SUCCESS) { - ssl::SSL_free_(ssl0); + sslGateway.SSL_free_(ssl0); std::string err = "Can not set host name for secure connection: " + GetSslError(ssl0, res); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, err.c_str()); } - ssl::SSL_set_connect_state_(ssl0); + sslGateway.SSL_set_connect_state_(ssl0); bool connected = CompleteConnectInternal(ssl0, timeout); if (!connected) { - ssl::SSL_free_(ssl0); + sslGateway.SSL_free_(ssl0); return false; } // Verify a server certificate was presented during the negotiation - X509* cert = ssl::SSL_get_peer_certificate(ssl0); + X509* cert = sslGateway.SSL_get_peer_certificate_(ssl0); if (cert) - ssl::X509_free(cert); + sslGateway.X509_free_(cert); else { - ssl::SSL_free_(ssl0); + sslGateway.SSL_free_(ssl0); // std::string err = "Remote host did not provide certificate: " + GetSslError(ssl0, res); @@ -113,10 +118,10 @@ namespace ignite // Verify the result of chain verification // Verification performed according to RFC 4158 - res = ssl::SSL_get_verify_result(ssl0); + res = sslGateway.SSL_get_verify_result_(ssl0); if (X509_V_OK != res) { - ssl::SSL_free_(ssl0); + sslGateway.SSL_free_(ssl0); // std::string err = "Certificate chain verification failed: " + GetSslError(ssl0, res); @@ -135,7 +140,9 @@ namespace ignite int SecureSocketClient::Send(const int8_t* data, size_t size, int32_t timeout) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (!ssl) { @@ -145,14 +152,16 @@ namespace ignite SSL* ssl0 = reinterpret_cast(ssl); - int res = ssl::SSL_write_(ssl0, data, static_cast(size)); + int res = sslGateway.SSL_write_(ssl0, data, static_cast(size)); return res; } int SecureSocketClient::Receive(int8_t* buffer, size_t size, int32_t timeout) { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (!ssl) { @@ -164,7 +173,7 @@ namespace ignite int res = 0; - if (!blocking && ssl::SSL_pending_(ssl0) == 0) + if (!blocking && sslGateway.SSL_pending_(ssl0) == 0) { res = WaitOnSocket(ssl, timeout, true); @@ -172,7 +181,7 @@ namespace ignite return res; } - res = ssl::SSL_read_(ssl0, buffer, static_cast(size)); + res = sslGateway.SSL_read_(ssl0, buffer, static_cast(size)); return res; } @@ -185,76 +194,58 @@ namespace ignite void* SecureSocketClient::MakeContext(const std::string& certPath, const std::string& keyPath, const std::string& caPath) { - assert(SslGateway::GetInstance().Loaded()); - - static bool sslLibInited = false; - static common::concurrent::CriticalSection sslCs; + SslGateway &sslGateway = SslGateway::GetInstance(); - if (!sslLibInited) - { - common::concurrent::CsLockGuard lock(sslCs); - - if (!sslLibInited) - { - (void)SSL_library_init(); + assert(sslGateway.Loaded()); - SSL_load_error_strings(); - - OPENSSL_config(0); - - sslLibInited = true; - } - } - - const SSL_METHOD* method = ssl::SSLv23_client_method_(); + const SSL_METHOD* method = sslGateway.SSLv23_client_method_(); if (!method) throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not get SSL method"); - SSL_CTX* ctx = ssl::SSL_CTX_new(method); + SSL_CTX* ctx = sslGateway.SSL_CTX_new_(method); if (!ctx) throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not create new SSL context"); - ssl::SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, 0); + sslGateway.SSL_CTX_set_verify_(ctx, SSL_VERIFY_PEER, 0); - ssl::SSL_CTX_set_verify_depth(ctx, 8); + sslGateway.SSL_CTX_set_verify_depth_(ctx, 8); - const long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION; - ssl::SSL_CTX_ctrl(ctx, SSL_CTRL_OPTIONS, flags, NULL); + sslGateway.SSL_CTX_set_options_(ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION); const char* cCaPath = caPath.empty() ? 0 : caPath.c_str(); - long res = ssl::SSL_CTX_load_verify_locations(ctx, cCaPath, 0); + long res = sslGateway.SSL_CTX_load_verify_locations_(ctx, cCaPath, 0); if (res != OPERATION_SUCCESS) { - ssl::SSL_CTX_free(ctx); + sslGateway.SSL_CTX_free_(ctx); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not set Certificate Authority path for secure connection"); } - res = ssl::SSL_CTX_use_certificate_chain_file(ctx, certPath.c_str()); + res = sslGateway.SSL_CTX_use_certificate_chain_file_(ctx, certPath.c_str()); if (res != OPERATION_SUCCESS) { - ssl::SSL_CTX_free(ctx); + sslGateway.SSL_CTX_free_(ctx); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not set client certificate file for secure connection"); } - res = ssl::SSL_CTX_use_RSAPrivateKey_file(ctx, keyPath.c_str(), SSL_FILETYPE_PEM); + res = sslGateway.SSL_CTX_use_RSAPrivateKey_file_(ctx, keyPath.c_str(), SSL_FILETYPE_PEM); if (res != OPERATION_SUCCESS) { - ssl::SSL_CTX_free(ctx); + sslGateway.SSL_CTX_free_(ctx); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not set private key file for secure connection"); } const char* const PREFERRED_CIPHERS = "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4"; - res = ssl::SSL_CTX_set_cipher_list(ctx, PREFERRED_CIPHERS); + res = sslGateway.SSL_CTX_set_cipher_list_(ctx, PREFERRED_CIPHERS); if (res != OPERATION_SUCCESS) { - ssl::SSL_CTX_free(ctx); + sslGateway.SSL_CTX_free_(ctx); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not set ciphers list for secure connection"); @@ -265,30 +256,34 @@ namespace ignite void* SecureSocketClient::MakeSsl(void* context, const char* hostname, uint16_t port, bool& blocking) { - BIO* bio = ssl::BIO_new_ssl_connect(reinterpret_cast(context)); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + + BIO* bio = sslGateway.BIO_new_ssl_connect_(reinterpret_cast(context)); if (!bio) throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not create SSL connection."); - blocking = ssl::BIO_set_nbio_(bio, 1) != OPERATION_SUCCESS; + blocking = sslGateway.BIO_set_nbio_(bio, 1) != OPERATION_SUCCESS; std::stringstream stream; stream << hostname << ":" << port; std::string address = stream.str(); - long res = ssl::BIO_set_conn_hostname_(bio, address.c_str()); + long res = sslGateway.BIO_set_conn_hostname_(bio, address.c_str()); if (res != OPERATION_SUCCESS) { - ssl::BIO_free_all(bio); + sslGateway.BIO_free_all_(bio); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not set SSL connection hostname."); } SSL* ssl = 0; - ssl::BIO_get_ssl_(bio, &ssl); + sslGateway.BIO_get_ssl_(bio, &ssl); if (!ssl) { - ssl::BIO_free_all(bio); + sslGateway.BIO_free_all_(bio); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "Can not get SSL instance from BIO."); } @@ -298,21 +293,25 @@ namespace ignite bool SecureSocketClient::CompleteConnectInternal(void* ssl, int timeout) { + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + SSL* ssl0 = reinterpret_cast(ssl); while (true) { - int res = ssl::SSL_connect_(ssl0); + int res = sslGateway.SSL_connect_(ssl0); if (res == OPERATION_SUCCESS) return true; - int sslError = ssl::SSL_get_error_(ssl0, res); + int sslError = sslGateway.SSL_get_error_(ssl0, res); if (IsActualError(sslError)) return false; - int want = ssl::SSL_want_(ssl0); + int want = sslGateway.SSL_want_(ssl0); res = WaitOnSocket(ssl, timeout, want == SSL_READING); @@ -326,9 +325,13 @@ namespace ignite std::string SecureSocketClient::GetSslError(void* ssl, int ret) { + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + SSL* ssl0 = reinterpret_cast(ssl); - int sslError = ssl::SSL_get_error_(ssl0, ret); + int sslError = sslGateway.SSL_get_error_(ssl0, ret); switch (sslError) { @@ -345,11 +348,11 @@ namespace ignite return std::string("SSL error: ") + common::LexicalCast(sslError); } - long error = ssl::ERR_get_error_(); + long error = sslGateway.ERR_get_error_(); char errBuf[1024] = { 0 }; - ssl::ERR_error_string_n_(error, errBuf, sizeof(errBuf)); + sslGateway.ERR_error_string_n_(error, errBuf, sizeof(errBuf)); return std::string(errBuf); } @@ -373,11 +376,13 @@ namespace ignite void SecureSocketClient::CloseInteral() { - assert(SslGateway::GetInstance().Loaded()); + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); if (ssl) { - ssl::SSL_free_(reinterpret_cast(ssl)); + sslGateway.SSL_free_(reinterpret_cast(ssl)); ssl = 0; } @@ -385,13 +390,17 @@ namespace ignite int SecureSocketClient::WaitOnSocket(void* ssl, int32_t timeout, bool rd) { + SslGateway &sslGateway = SslGateway::GetInstance(); + + assert(sslGateway.Loaded()); + int ready = 0; int lastError = 0; SSL* ssl0 = reinterpret_cast(ssl); fd_set fds; - int fd = ssl::SSL_get_fd_(ssl0); + int fd = sslGateway.SSL_get_fd_(ssl0); if (fd < 0) { diff --git a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.cpp b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.cpp new file mode 100644 index 0000000000000..a1da21524d6f1 --- /dev/null +++ b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "impl/ssl/ssl_gateway.h" +#include "impl/ssl/ssl_api.h" + +namespace ignite +{ + namespace impl + { + namespace thin + { + namespace ssl + { + void EnsureSslLoaded() + { + SslGateway::GetInstance().LoadAll(); + } + } + } + } +} diff --git a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.h b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.h new file mode 100644 index 0000000000000..8d6b72dbd00f6 --- /dev/null +++ b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_api.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _IGNITE_IMPL_THIN_SSL_SSL_API +#define _IGNITE_IMPL_THIN_SSL_SSL_API + + +namespace ignite +{ + namespace impl + { + namespace thin + { + namespace ssl + { + void EnsureSslLoaded(); + } + } + } +} + +#endif //_IGNITE_IMPL_THIN_SSL_SSL_API \ No newline at end of file diff --git a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_bindings.h b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_bindings.h deleted file mode 100644 index 05ad00d9354ce..0000000000000 --- a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_bindings.h +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _IGNITE_IMPL_THIN_SSL_SSL_BINDINGS -#define _IGNITE_IMPL_THIN_SSL_SSL_BINDINGS - -#include -#include -#include - -#include "impl/ssl/ssl_gateway.h" - -namespace ignite -{ - namespace impl - { - namespace thin - { - namespace ssl - { - // Declaring constant used by OpenSSL for readability. - enum { OPERATION_SUCCESS = 1 }; - - inline SSL_CTX *SSL_CTX_new(const SSL_METHOD *meth) - { - typedef SSL_CTX*(FuncType)(const SSL_METHOD*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_CTX_new); - - return fp(meth); - } - - inline void SSL_CTX_free(SSL_CTX *ctx) - { - typedef void(FuncType)(SSL_CTX*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_CTX_free); - - fp(ctx); - } - - inline void SSL_CTX_set_verify(SSL_CTX *ctx, int mode, int(*callback) (int, X509_STORE_CTX *)) - { - typedef void(FuncType)(SSL_CTX*, int, int(*)(int, X509_STORE_CTX*)); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_set_verify); - - fp(ctx, mode, callback); - } - - inline void SSL_CTX_set_verify_depth(SSL_CTX *ctx, int depth) - { - typedef void(FuncType)(SSL_CTX*, int); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_set_verify_depth); - - fp(ctx, depth); - } - - inline int SSL_CTX_load_verify_locations(SSL_CTX *ctx, const char *cAfile, const char *cApath) - { - typedef int(FuncType)(SSL_CTX*, const char*, const char*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_load_verify_locations); - - return fp(ctx, cAfile, cApath); - } - - inline int SSL_CTX_use_certificate_chain_file(SSL_CTX *ctx, const char *file) - { - typedef int(FuncType)(SSL_CTX*, const char*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_use_certificate_chain_file); - - return fp(ctx, file); - } - - inline int SSL_CTX_use_RSAPrivateKey_file(SSL_CTX *ctx, const char *file, int type) - { - typedef int(FuncType)(SSL_CTX*, const char*, int); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_use_RSAPrivateKey_file); - - return fp(ctx, file, type); - } - - inline int SSL_CTX_set_cipher_list(SSL_CTX *ctx, const char *str) - { - typedef int(FuncType)(SSL_CTX*, const char*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_CTX_set_cipher_list); - - return fp(ctx, str); - } - - inline long SSL_CTX_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) - { - typedef long(FuncType)(SSL_CTX*, int, long, void*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_CTX_ctrl); - - return fp(ctx, cmd, larg, parg); - } - - inline long SSL_get_verify_result(const SSL *s) - { - typedef long(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_get_verify_result); - - return fp(s); - } - - inline int SSL_library_init() - { - typedef int(FuncType)(); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_library_init); - - return fp(); - } - - inline void SSL_load_error_strings() - { - typedef void(FuncType)(); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_load_error_strings); - - fp(); - } - - inline X509 *SSL_get_peer_certificate(const SSL *s) - { - typedef X509*(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_get_peer_certificate); - - return fp(s); - } - - inline long SSL_ctrl(SSL *s, int cmd, long larg, void *parg) - { - typedef long(FuncType)(SSL*, int, long, void*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_ctrl); - - return fp(s, cmd, larg ,parg); - } - - inline long SSL_set_tlsext_host_name_(SSL *s, const char *name) - { - return ssl::SSL_ctrl(s, SSL_CTRL_SET_TLSEXT_HOSTNAME, - TLSEXT_NAMETYPE_host_name, const_cast(name)); - } - - inline void SSL_set_connect_state_(SSL* s) - { - typedef void(FuncType)(SSL*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSL_set_connect_state); - - return fp(s); - } - - inline int SSL_connect_(SSL* s) - { - typedef int(FuncType)(SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_connect); - - return fp(s); - } - - inline int SSL_get_error_(const SSL *s, int ret) - { - typedef int(FuncType)(const SSL*, int); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_get_error); - - return fp(s, ret); - } - - inline int SSL_want_(const SSL *s) - { - typedef int(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_want); - - return fp(s); - } - - inline int SSL_write_(SSL *s, const void *buf, int num) - { - typedef int(FuncType)(SSL*, const void*, int); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_write); - - return fp(s, buf, num); - } - - inline int SSL_read_(SSL *s, void *buf, int num) - { - typedef int(FuncType)(SSL*, void*, int); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_read); - - return fp(s, buf, num); - } - - inline int SSL_pending_(const SSL *ssl) - { - typedef int(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_pending); - - return fp(ssl); - } - - inline int SSL_get_fd_(const SSL *ssl) - { - typedef int(FuncType)(const SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_get_fd); - - return fp(ssl); - } - - inline void SSL_free_(SSL *ssl) - { - typedef void(FuncType)(SSL*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpSSL_free); - - fp(ssl); - } - - inline const SSL_METHOD *SSLv23_client_method_() - { - typedef const SSL_METHOD*(FuncType)(); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpSSLv23_client_method); - - return fp(); - } - - inline void OPENSSL_config(const char *configName) - { - typedef void(FuncType)(const char*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpOPENSSL_config); - - fp(configName); - } - - inline void X509_free(X509 *a) - { - typedef void(FuncType)(X509*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpX509_free); - - fp(a); - } - - inline BIO *BIO_new_ssl_connect(SSL_CTX *ctx) - { - typedef BIO*(FuncType)(SSL_CTX*); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpBIO_new_ssl_connect); - - return fp(ctx); - } - - inline void BIO_free_all(BIO *a) - { - typedef void(FuncType)(BIO*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpBIO_free_all); - - fp(a); - } - - inline long BIO_ctrl(BIO *bp, int cmd, long larg, void *parg) - { - typedef long(FuncType)(BIO*, int, long, void*); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpBIO_ctrl); - - return fp(bp, cmd, larg, parg); - } - - inline long BIO_get_fd_(BIO *bp, int *fd) - { - return ssl::BIO_ctrl(bp, BIO_C_GET_FD, 0, reinterpret_cast(fd)); - } - - inline long BIO_get_ssl_(BIO *bp, SSL** ssl) - { - return ssl::BIO_ctrl(bp, BIO_C_GET_SSL, 0, reinterpret_cast(ssl)); - } - - inline long BIO_set_nbio_(BIO *bp, long n) - { - return ssl::BIO_ctrl(bp, BIO_C_SET_NBIO, n, NULL); - } - - inline long BIO_set_conn_hostname_(BIO *bp, const char *name) - { - return ssl::BIO_ctrl(bp, BIO_C_SET_CONNECT, 0, const_cast(name)); - } - - inline unsigned long ERR_get_error_() - { - typedef unsigned long(FuncType)(); - - FuncType* fp = reinterpret_cast(SslGateway::GetInstance().GetFunctions().fpERR_get_error); - - return fp(); - } - - inline void ERR_error_string_n_(unsigned long e, char *buf, size_t len) - { - typedef void(FuncType)(unsigned long, char*, size_t); - - FuncType* fp = reinterpret_cast( - SslGateway::GetInstance().GetFunctions().fpERR_error_string_n); - - fp(e, buf, len); - } - } - } - } -} - -#endif //_IGNITE_IMPL_THIN_SSL_SSL_BINDINGS \ No newline at end of file diff --git a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.cpp b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.cpp index 380fe0f1843fd..42ed88dc7d47c 100644 --- a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.cpp +++ b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.cpp @@ -26,6 +26,18 @@ # define ADDITIONAL_OPENSSL_HOME_ENV "OPEN_SSL_HOME" #endif // ADDITIONAL_OPENSSL_HOME_ENV +#ifndef SSL_CTRL_OPTIONS +# define SSL_CTRL_OPTIONS 32 +#endif // SSL_CTRL_OPTIONS + +#ifndef OPENSSL_INIT_LOAD_SSL_STRINGS +# define OPENSSL_INIT_LOAD_SSL_STRINGS 0x00200000L +#endif // OPENSSL_INIT_LOAD_SSL_STRINGS + +#ifndef OPENSSL_INIT_LOAD_CRYPTO_STRINGS +# define OPENSSL_INIT_LOAD_CRYPTO_STRINGS 0x00000002L +#endif // OPENSSL_INIT_LOAD_CRYPTO_STRINGS + namespace ignite { namespace impl @@ -38,7 +50,7 @@ namespace ignite inited(false), functions() { - // No-op. + memset(&functions, 0, sizeof(functions)); } SslGateway::~SslGateway() @@ -46,41 +58,61 @@ namespace ignite // No-op. } + void SslGateway::UnloadAll() + { + libeay32.Unload(); + ssleay32.Unload(); + libssl.Unload(); + libcrypto.Unload(); + + memset(&functions, 0, sizeof(functions)); + } + common::dynamic::Module SslGateway::LoadSslLibrary(const char* name) { using namespace common; using namespace dynamic; - std::string fullName = GetDynamicLibraryName(name); - - Module libModule = LoadModule(fullName); - - if (libModule.IsLoaded()) - return libModule; - std::string home = GetEnv(ADDITIONAL_OPENSSL_HOME_ENV); if (home.empty()) home = GetEnv("OPENSSL_HOME"); - if (home.empty()) - return libModule; + std::string fullName = GetDynamicLibraryName(name); + + if (!home.empty()) + { + std::stringstream constructor; - std::stringstream constructor; + constructor << home << Fs << "bin" << Fs << fullName; - constructor << home << Fs << "bin" << Fs << fullName; + std::string fullPath = constructor.str(); - std::string fullPath = constructor.str(); + Module mod = LoadModule(fullPath); - return LoadModule(fullPath); + if (mod.IsLoaded()) + return mod; + } + + return LoadModule(fullName); } void SslGateway::LoadSslLibraries() { - libeay32 = LoadSslLibrary("libeay32"); - ssleay32 = LoadSslLibrary("ssleay32"); libssl = LoadSslLibrary("libssl"); + if (!libssl.IsLoaded()) + { + libcrypto = LoadSslLibrary("libcrypto-1_1-x64"); + libssl = LoadSslLibrary("libssl-1_1-x64"); + } + + if (!libssl.IsLoaded()) + { + libeay32 = LoadSslLibrary("libeay32"); + ssleay32 = LoadSslLibrary("ssleay32"); + } + if (!libssl.IsLoaded() && (!libeay32.IsLoaded() || !ssleay32.IsLoaded())) { if (!libssl.IsLoaded()) @@ -89,44 +121,37 @@ namespace ignite std::stringstream ss; - ss << "Can not load neccessary OpenSSL libraries: "; + ss << "Can not load neccessary OpenSSL libraries:"; if (!libeay32.IsLoaded()) - ss << "libeay32"; + ss << " libeay32"; if (!ssleay32.IsLoaded()) ss << " ssleay32"; - libeay32.Unload(); - ssleay32.Unload(); - libssl.Unload(); - std::string res = ss.str(); throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, res.c_str()); } } - SslGateway& SslGateway::GetInstance() + void SslGateway::LoadMandatoryMethods() { - static SslGateway self; + functions.fpSSLeay_version = TryLoadSslMethod("SSLeay_version"); - return self; - } - - void SslGateway::LoadAll() - { - using namespace common::dynamic; + if (!functions.fpSSLeay_version) + functions.fpOpenSSL_version = LoadSslMethod("OpenSSL_version"); - if (inited) - return; + functions.fpSSL_library_init = TryLoadSslMethod("SSL_library_init"); + functions.fpSSL_load_error_strings = TryLoadSslMethod("SSL_load_error_strings"); - common::concurrent::CsLockGuard lock(initCs); + if (!functions.fpSSL_library_init || !functions.fpSSL_load_error_strings) + functions.fpOPENSSL_init_ssl = LoadSslMethod("OPENSSL_init_ssl"); - if (inited) - return; + functions.fpSSLv23_client_method = TryLoadSslMethod("SSLv23_client_method"); - LoadSslLibraries(); + if (!functions.fpSSLv23_client_method) + functions.fpTLS_client_method = LoadSslMethod("TLS_client_method"); functions.fpSSL_CTX_new = LoadSslMethod("SSL_CTX_new"); functions.fpSSL_CTX_free = LoadSslMethod("SSL_CTX_free"); @@ -138,13 +163,11 @@ namespace ignite functions.fpSSL_CTX_set_cipher_list = LoadSslMethod("SSL_CTX_set_cipher_list"); functions.fpSSL_get_verify_result = LoadSslMethod("SSL_get_verify_result"); - functions.fpSSL_library_init = LoadSslMethod("SSL_library_init"); - functions.fpSSL_load_error_strings = LoadSslMethod("SSL_load_error_strings"); + functions.fpSSL_get_peer_certificate = LoadSslMethod("SSL_get_peer_certificate"); functions.fpSSL_ctrl = LoadSslMethod("SSL_ctrl"); functions.fpSSL_CTX_ctrl = LoadSslMethod("SSL_CTX_ctrl"); - functions.fpSSLv23_client_method = LoadSslMethod("SSLv23_client_method"); functions.fpSSL_set_connect_state = LoadSslMethod("SSL_set_connect_state"); functions.fpSSL_connect = LoadSslMethod("SSL_connect"); functions.fpSSL_get_error = LoadSslMethod("SSL_get_error"); @@ -164,60 +187,67 @@ namespace ignite functions.fpERR_get_error = LoadSslMethod("ERR_get_error"); functions.fpERR_error_string_n = LoadSslMethod("ERR_error_string_n"); + } - bool allLoaded = - functions.fpSSL_CTX_new != 0 && - functions.fpSSL_CTX_free != 0 && - functions.fpSSL_CTX_set_verify != 0 && - functions.fpSSL_CTX_set_verify_depth != 0 && - functions.fpSSL_CTX_load_verify_locations != 0 && - functions.fpSSL_CTX_use_certificate_chain_file != 0 && - functions.fpSSL_CTX_use_RSAPrivateKey_file != 0 && - functions.fpSSL_CTX_set_cipher_list != 0 && - functions.fpSSL_get_verify_result != 0 && - functions.fpSSL_library_init != 0 && - functions.fpSSL_load_error_strings != 0 && - functions.fpSSL_get_peer_certificate != 0 && - functions.fpSSL_ctrl != 0 && - functions.fpSSL_CTX_ctrl != 0 && - functions.fpSSLv23_client_method != 0 && - functions.fpSSL_set_connect_state != 0 && - functions.fpSSL_connect != 0 && - functions.fpSSL_get_error != 0 && - functions.fpSSL_want != 0 && - functions.fpSSL_write != 0 && - functions.fpSSL_read != 0 && - functions.fpSSL_pending != 0 && - functions.fpSSL_get_fd != 0 && - functions.fpSSL_free != 0 && - functions.fpBIO_new_ssl_connect != 0 && - functions.fpOPENSSL_config != 0 && - functions.fpX509_free != 0 && - functions.fpBIO_free_all != 0 && - functions.fpBIO_ctrl != 0 && - functions.fpERR_get_error != 0 && - functions.fpERR_error_string_n != 0; - - if (!allLoaded) - { - libeay32.Unload(); - ssleay32.Unload(); - libssl.Unload(); - } + SslGateway& SslGateway::GetInstance() + { + static SslGateway self; - inited = allLoaded; + return self; } - void* SslGateway::LoadSslMethod(const char* name) + void SslGateway::LoadAll() + { + using namespace common::dynamic; + + if (inited) + return; + + common::concurrent::CsLockGuard lock(initCs); + + if (inited) + return; + + common::MethodGuard guard(this, &SslGateway::UnloadAll); + + LoadSslLibraries(); + + LoadMandatoryMethods(); + + functions.fpSSL_CTX_set_options = TryLoadSslMethod("SSL_CTX_set_options"); + functions.fpERR_print_errors_fp = TryLoadSslMethod("ERR_print_errors_fp"); + + (void)SSL_library_init_(); + + SSL_load_error_strings_(); + + OPENSSL_config_(0); + + guard.Release(); + + inited = true; + } + + void* SslGateway::TryLoadSslMethod(const char* name) { void* fp = libeay32.FindSymbol(name); if (!fp) fp = ssleay32.FindSymbol(name); + if (!fp) + fp = libcrypto.FindSymbol(name); + if (!fp) fp = libssl.FindSymbol(name); + return fp; + } + + void* SslGateway::LoadSslMethod(const char* name) + { + void* fp = TryLoadSslMethod(name); + if (!fp) { std::stringstream ss; @@ -231,6 +261,448 @@ namespace ignite return fp; } + + char* SslGateway::SSLeay_version_(int type) + { + typedef char* (FuncType)(int); + + FuncType* fp = 0; + + if (functions.fpSSLeay_version) + fp = reinterpret_cast(functions.fpSSLeay_version); + else + fp = reinterpret_cast(functions.fpOpenSSL_version); + + assert(fp != 0); + + return fp(type); + } + + int SslGateway::OPENSSL_init_ssl_(uint64_t opts, const void* settings) + { + assert(functions.fpOPENSSL_init_ssl != 0); + + typedef int (FuncType)(uint64_t, const void*); + + FuncType* fp = reinterpret_cast(functions.fpOPENSSL_init_ssl); + + return fp(opts, settings); + } + + long SslGateway::SSL_CTX_set_options_(SSL_CTX* ctx, long options) + { + if (functions.fpSSL_CTX_set_options) + { + typedef long (FuncType)(SSL_CTX*, long); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_options); + + return fp(ctx, options); + } + + return SSL_CTX_ctrl_(ctx, SSL_CTRL_OPTIONS, options, NULL); + } + + long SslGateway::SSL_CTX_ctrl_(SSL_CTX* ctx, int cmd, long larg, void* parg) + { + assert(functions.fpSSL_CTX_ctrl != 0); + + typedef long (FuncType)(SSL_CTX*, int, long, void*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_ctrl); + + return fp(ctx, cmd, larg, parg); + } + + SSL_CTX* SslGateway::SSL_CTX_new_(const SSL_METHOD* meth) + { + assert(functions.fpSSL_CTX_new != 0); + + typedef SSL_CTX*(FuncType)(const SSL_METHOD*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_new); + + return fp(meth); + } + + void SslGateway::SSL_CTX_free_(SSL_CTX* ctx) + { + assert(functions.fpSSL_CTX_free != 0); + + typedef void (FuncType)(SSL_CTX*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_free); + + fp(ctx); + } + + void SslGateway::SSL_CTX_set_verify_(SSL_CTX* ctx, int mode, int (* callback)(int, X509_STORE_CTX*)) + { + assert(functions.fpSSL_CTX_set_verify != 0); + + typedef void (FuncType)(SSL_CTX*, int, int (*)(int, X509_STORE_CTX*)); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_verify); + + fp(ctx, mode, callback); + } + + void SslGateway::SSL_CTX_set_verify_depth_(SSL_CTX* ctx, int depth) + { + assert(functions.fpSSL_CTX_set_verify_depth != 0); + + typedef void (FuncType)(SSL_CTX*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_verify_depth); + + fp(ctx, depth); + } + + int SslGateway::SSL_CTX_load_verify_locations_(SSL_CTX* ctx, const char* cAfile, const char* cApath) + { + assert(functions.fpSSL_CTX_load_verify_locations != 0); + + typedef int (FuncType)(SSL_CTX*, const char*, const char*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_load_verify_locations); + + return fp(ctx, cAfile, cApath); + } + + int SslGateway::SSL_CTX_use_certificate_chain_file_(SSL_CTX* ctx, const char* file) + { + assert(functions.fpSSL_CTX_use_certificate_chain_file != 0); + + typedef int (FuncType)(SSL_CTX*, const char*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_use_certificate_chain_file); + + return fp(ctx, file); + } + + int SslGateway::SSL_CTX_use_RSAPrivateKey_file_(SSL_CTX* ctx, const char* file, int type) + { + assert(functions.fpSSL_CTX_use_RSAPrivateKey_file != 0); + + typedef int (FuncType)(SSL_CTX*, const char*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_use_RSAPrivateKey_file); + + return fp(ctx, file, type); + } + + int SslGateway::SSL_CTX_set_cipher_list_(SSL_CTX* ctx, const char* str) + { + assert(functions.fpSSL_CTX_set_cipher_list != 0); + + typedef int (FuncType)(SSL_CTX*, const char*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_CTX_set_cipher_list); + + return fp(ctx, str); + } + + long SslGateway::SSL_get_verify_result_(const SSL* s) + { + assert(functions.fpSSL_get_verify_result != 0); + + typedef long (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_verify_result); + + return fp(s); + } + + int SslGateway::SSL_library_init_() + { + typedef int (FuncType)(); + + if (functions.fpSSL_library_init) + { + FuncType* fp = reinterpret_cast(functions.fpSSL_library_init); + + return fp(); + } + + return OPENSSL_init_ssl_(0, NULL); + } + + void SslGateway::SSL_load_error_strings_() + { + typedef void (FuncType)(); + + if (functions.fpSSL_load_error_strings) + { + FuncType* fp = reinterpret_cast(functions.fpSSL_load_error_strings); + + fp(); + + return; + } + + OPENSSL_init_ssl_(OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); + } + + X509* SslGateway::SSL_get_peer_certificate_(const SSL* s) + { + assert(functions.fpSSL_get_peer_certificate != 0); + + typedef X509*(FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_peer_certificate); + + return fp(s); + } + + long SslGateway::SSL_ctrl_(SSL* s, int cmd, long larg, void* parg) + { + assert(functions.fpSSL_ctrl != 0); + + typedef long (FuncType)(SSL*, int, long, void*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_ctrl); + + return fp(s, cmd, larg, parg); + } + + long SslGateway::SSL_set_tlsext_host_name_(SSL* s, const char* name) + { + return SSL_ctrl_(s, SSL_CTRL_SET_TLSEXT_HOSTNAME, + TLSEXT_NAMETYPE_host_name, const_cast(name)); + } + + void SslGateway::SSL_set_connect_state_(SSL* s) + { + assert(functions.fpSSL_set_connect_state != 0); + + typedef void (FuncType)(SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_set_connect_state); + + return fp(s); + } + + int SslGateway::SSL_connect_(SSL* s) + { + assert(functions.fpSSL_connect != 0); + + typedef int (FuncType)(SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_connect); + + return fp(s); + } + + int SslGateway::SSL_get_error_(const SSL* s, int ret) + { + assert(functions.fpSSL_get_error != 0); + + typedef int (FuncType)(const SSL*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_error); + + return fp(s, ret); + } + + int SslGateway::SSL_want_(const SSL* s) + { + assert(functions.fpSSL_want != 0); + + typedef int (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_want); + + return fp(s); + } + + int SslGateway::SSL_write_(SSL* s, const void* buf, int num) + { + assert(functions.fpSSL_write != 0); + + typedef int (FuncType)(SSL*, const void*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_write); + + return fp(s, buf, num); + } + + int SslGateway::SSL_read_(SSL* s, void* buf, int num) + { + assert(functions.fpSSL_read != 0); + + typedef int (FuncType)(SSL*, void*, int); + + FuncType* fp = reinterpret_cast(functions.fpSSL_read); + + return fp(s, buf, num); + } + + int SslGateway::SSL_pending_(const SSL* ssl) + { + assert(functions.fpSSL_pending != 0); + + typedef int (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_pending); + + return fp(ssl); + } + + int SslGateway::SSL_get_fd_(const SSL* ssl) + { + assert(functions.fpSSL_get_fd != 0); + + typedef int (FuncType)(const SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_get_fd); + + return fp(ssl); + } + + void SslGateway::SSL_free_(SSL* ssl) + { + assert(functions.fpSSL_free != 0); + + typedef void (FuncType)(SSL*); + + FuncType* fp = reinterpret_cast(functions.fpSSL_free); + + fp(ssl); + } + + const SSL_METHOD* SslGateway::SSLv23_client_method_() + { + if (functions.fpSSLv23_client_method) + { + typedef const SSL_METHOD*(FuncType)(); + + FuncType* fp = reinterpret_cast(functions.fpSSLv23_client_method); + + return fp(); + } + + return TLS_client_method_(); + } + + const SSL_METHOD* SslGateway::TLS_client_method_() + { + assert(functions.fpTLS_client_method != 0); + + typedef const SSL_METHOD*(FuncType)(); + + FuncType* fp = reinterpret_cast(functions.fpTLS_client_method); + + return fp(); + } + + void SslGateway::OPENSSL_config_(const char* configName) + { + assert(functions.fpOPENSSL_config != 0); + + typedef void (FuncType)(const char*); + + FuncType* fp = reinterpret_cast(functions.fpOPENSSL_config); + + fp(configName); + } + + void SslGateway::X509_free_(X509* a) + { + assert(functions.fpX509_free != 0); + + typedef void (FuncType)(X509*); + + FuncType* fp = reinterpret_cast(functions.fpX509_free); + + fp(a); + } + + BIO* SslGateway::BIO_new_ssl_connect_(SSL_CTX* ctx) + { + assert(functions.fpBIO_new_ssl_connect != 0); + + typedef BIO*(FuncType)(SSL_CTX*); + + FuncType* fp = reinterpret_cast(functions.fpBIO_new_ssl_connect); + + return fp(ctx); + } + + void SslGateway::BIO_free_all_(BIO* a) + { + assert(functions.fpBIO_free_all != 0); + + typedef void (FuncType)(BIO*); + + FuncType* fp = reinterpret_cast(functions.fpBIO_free_all); + + fp(a); + } + + long SslGateway::BIO_ctrl_(BIO* bp, int cmd, long larg, void* parg) + { + assert(functions.fpBIO_ctrl != 0); + + typedef long (FuncType)(BIO*, int, long, void*); + + FuncType* fp = reinterpret_cast(functions.fpBIO_ctrl); + + return fp(bp, cmd, larg, parg); + } + + long SslGateway::BIO_get_fd_(BIO* bp, int* fd) + { + return BIO_ctrl_(bp, BIO_C_GET_FD, 0, reinterpret_cast(fd)); + } + + long SslGateway::BIO_get_ssl_(BIO* bp, SSL** ssl) + { + return BIO_ctrl_(bp, BIO_C_GET_SSL, 0, reinterpret_cast(ssl)); + } + + long SslGateway::BIO_set_nbio_(BIO* bp, long n) + { + return BIO_ctrl_(bp, BIO_C_SET_NBIO, n, NULL); + } + + long SslGateway::BIO_set_conn_hostname_(BIO* bp, const char* name) + { + return BIO_ctrl_(bp, BIO_C_SET_CONNECT, 0, const_cast(name)); + } + + unsigned long SslGateway::ERR_get_error_() + { + assert(functions.fpERR_get_error != 0); + + typedef unsigned long (FuncType)(); + + FuncType* fp = reinterpret_cast(functions.fpERR_get_error); + + return fp(); + } + + void SslGateway::ERR_error_string_n_(unsigned long e, char* buf, size_t len) + { + assert(functions.fpERR_error_string_n != 0); + + typedef void (FuncType)(unsigned long, char*, size_t); + + FuncType* fp = reinterpret_cast(functions.fpERR_error_string_n); + + fp(e, buf, len); + } + + void SslGateway::ERR_print_errors_fp_(FILE* fd) + { + if (!functions.fpERR_print_errors_fp) + return; + + typedef void (FuncType)(FILE*); + + FuncType* fp = reinterpret_cast(functions.fpERR_print_errors_fp); + + fp(fd); + } } } } diff --git a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.h b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.h index 440aaa867c6f9..0908c103581c1 100644 --- a/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.h +++ b/modules/platforms/cpp/thin-client/src/impl/ssl/ssl_gateway.h @@ -18,9 +18,14 @@ #ifndef _IGNITE_IMPL_THIN_SSL_SSL_LIBRARY #define _IGNITE_IMPL_THIN_SSL_SSL_LIBRARY +#include +#include +#include + #include #include + namespace ignite { namespace impl @@ -34,6 +39,7 @@ namespace ignite */ struct SslFunctions { + void *fpSSLeay_version; void *fpSSL_CTX_new; void *fpSSL_CTX_free; void *fpSSL_CTX_set_verify; @@ -65,6 +71,12 @@ namespace ignite void *fpBIO_ctrl; void *fpERR_get_error; void *fpERR_error_string_n; + void *fpERR_print_errors_fp; + + void *fpOpenSSL_version; + void *fpSSL_CTX_set_options; + void *fpOPENSSL_init_ssl; + void *fpTLS_client_method; }; /** @@ -104,6 +116,88 @@ namespace ignite return inited; } + char* SSLeay_version_(int type); + + int OPENSSL_init_ssl_(uint64_t opts, const void* settings); + + long SSL_CTX_set_options_(SSL_CTX* ctx, long options); + + long SSL_CTX_ctrl_(SSL_CTX* ctx, int cmd, long larg, void* parg); + + SSL_CTX* SSL_CTX_new_(const SSL_METHOD* meth); + + void SSL_CTX_free_(SSL_CTX* ctx); + + void SSL_CTX_set_verify_(SSL_CTX* ctx, int mode, int (*callback)(int, X509_STORE_CTX*)); + + void SSL_CTX_set_verify_depth_(SSL_CTX* ctx, int depth); + + int SSL_CTX_load_verify_locations_(SSL_CTX* ctx, const char* cAfile, const char* cApath); + + int SSL_CTX_use_certificate_chain_file_(SSL_CTX* ctx, const char* file); + + int SSL_CTX_use_RSAPrivateKey_file_(SSL_CTX* ctx, const char* file, int type); + + int SSL_CTX_set_cipher_list_(SSL_CTX* ctx, const char* str); + + long SSL_get_verify_result_(const SSL* s); + + int SSL_library_init_(); + + void SSL_load_error_strings_(); + + X509* SSL_get_peer_certificate_(const SSL* s); + + long SSL_ctrl_(SSL* s, int cmd, long larg, void* parg); + + long SSL_set_tlsext_host_name_(SSL* s, const char* name); + + void SSL_set_connect_state_(SSL* s); + + int SSL_connect_(SSL* s); + + int SSL_get_error_(const SSL* s, int ret); + + int SSL_want_(const SSL* s); + + int SSL_write_(SSL* s, const void* buf, int num); + + int SSL_read_(SSL* s, void* buf, int num); + + int SSL_pending_(const SSL* ssl); + + int SSL_get_fd_(const SSL* ssl); + + void SSL_free_(SSL* ssl); + + const SSL_METHOD* SSLv23_client_method_(); + + const SSL_METHOD* TLS_client_method_(); + + void OPENSSL_config_(const char* configName); + + void X509_free_(X509* a); + + BIO* BIO_new_ssl_connect_(SSL_CTX* ctx); + + void BIO_free_all_(BIO* a); + + long BIO_ctrl_(BIO* bp, int cmd, long larg, void* parg); + + long BIO_get_fd_(BIO* bp, int* fd); + + long BIO_get_ssl_(BIO* bp, SSL** ssl); + + long BIO_set_nbio_(BIO* bp, long n); + + long BIO_set_conn_hostname_(BIO* bp, const char* name); + + unsigned long ERR_get_error_(); + + void ERR_error_string_n_(unsigned long e, char* buf, size_t len); + + void ERR_print_errors_fp_(FILE *fd); + private: /** * Constructor. @@ -115,6 +209,11 @@ namespace ignite */ ~SslGateway(); + /** + * Unload all SSL symbols. + */ + void UnloadAll(); + /** * Load SSL library. * @param name Name. @@ -127,10 +226,28 @@ namespace ignite */ void LoadSslLibraries(); + /** + * Load mandatory SSL methods. + * + * @throw IgniteError if can not load one of the functions. + */ + void LoadMandatoryMethods(); + + /** + * Try load SSL method. + * + * @param name Name. + * @return Method pointer. + */ + void* TryLoadSslMethod(const char* name); + /** * Load SSL method. + * * @param name Name. * @return Method pointer. + * + * @throw IgniteError if the method is not present. */ void* LoadSslMethod(const char* name); @@ -146,6 +263,9 @@ namespace ignite /** ssleay32 module. */ common::dynamic::Module ssleay32; + /** libcrypto module. */ + common::dynamic::Module libcrypto; + /** libssl module. */ common::dynamic::Module libssl; From d3670a654673f3cff6bbfb31a6fe37ca7c74e97c Mon Sep 17 00:00:00 2001 From: YuriBabak Date: Tue, 11 Dec 2018 15:20:43 +0300 Subject: [PATCH 223/403] IGNITE-10630: Missed descriptions for ML packages This closes #5636 (cherry picked from commit e16a6e1) --- .../preprocessing/package-info.java | 22 +++++++++++++++++++ .../scoring/evaluator/package-info.java | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/preprocessing/package-info.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/selection/scoring/evaluator/package-info.java diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/preprocessing/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/preprocessing/package-info.java new file mode 100644 index 0000000000000..77dd980c56a0d --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/preprocessing/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains exceptions for preprocessing. + */ +package org.apache.ignite.ml.math.exceptions.preprocessing; \ No newline at end of file diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/selection/scoring/evaluator/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/selection/scoring/evaluator/package-info.java new file mode 100644 index 0000000000000..c2c7c4307ba5e --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/selection/scoring/evaluator/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Package for model evaluator classes. + */ +package org.apache.ignite.ml.selection.scoring.evaluator; \ No newline at end of file From d402bb7cacc67291655570536e9320ddd7835184 Mon Sep 17 00:00:00 2001 From: Anton Kalashnikov Date: Tue, 11 Dec 2018 15:39:42 +0300 Subject: [PATCH 224/403] IGNITE-10622 Fixed missed discovery event in the case of the next node failure - Fixes #5628. Signed-off-by: Alexey Goncharuk --- .../ignite/spi/discovery/tcp/ServerImpl.java | 2 +- ...cpDiscoveryPendingMessageDeliveryTest.java | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index fa9033d626fed..a643b44282c92 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -3227,7 +3227,7 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof assert !forceSndPending || msg instanceof TcpDiscoveryNodeLeftMessage; - if (failure || forceSndPending) { + if (failure || forceSndPending || newNextNode) { if (log.isDebugEnabled()) log.debug("Pending messages will be sent [failure=" + failure + ", newNextNode=" + newNextNode + diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java index 9b3dfeea2e93d..9ebf19f983517 100644 --- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java @@ -24,11 +24,13 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper; import org.apache.ignite.internal.managers.discovery.DiscoCache; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; @@ -72,6 +74,8 @@ public class TcpDiscoveryPendingMessageDeliveryTest extends GridCommonAbstractTe disco = new DyingDiscoverySpi(); else if (igniteInstanceName.startsWith("listener")) disco = new ListeningDiscoverySpi(); + else if (igniteInstanceName.startsWith("receiver")) + disco = new DyingThreadDiscoverySpi(); else disco = new TcpDiscoverySpi(); @@ -184,6 +188,47 @@ public void testCustomMessageInSingletonCluster() throws Exception { }, 10000)); } + /** + * @throws Exception If failed. + */ + public void testDeliveryAllFailedMessagesInCorrectOrder() throws Exception { + IgniteEx coord = startGrid("coordinator"); + TcpDiscoverySpi coordDisco = (TcpDiscoverySpi)coord.configuration().getDiscoverySpi(); + + Set sentEnsuredMsgs = new GridConcurrentHashSet<>(); + coordDisco.addSendMessageListener(msg -> { + if (coordDisco.ensured(msg)) + sentEnsuredMsgs.add(msg); + }); + + //Node which receive message but will not send it further around the ring. + IgniteEx receiver = startGrid("receiver"); + + //Node which will be failed first. + IgniteEx dummy = startGrid("dummy"); + + //Node which should received all fail message in any way. + startGrid("listener"); + + sentEnsuredMsgs.clear(); + receivedEnsuredMsgs.clear(); + + blockMsgs = true; + + log.info("Sending fail node messages"); + + coord.context().discovery().failNode(dummy.localNode().id(), "Dummy node failed"); + coord.context().discovery().failNode(receiver.localNode().id(), "Receiver node failed"); + + boolean delivered = GridTestUtils.waitForCondition(() -> { + log.info("Waiting for messages delivery"); + + return receivedEnsuredMsgs.equals(sentEnsuredMsgs); + }, 5000); + + assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs, delivered); + } + /** * @param disco Discovery SPI. * @param id Message id. @@ -192,6 +237,17 @@ private void sendDummyCustomMessage(TcpDiscoverySpi disco, IgniteUuid id) { disco.sendCustomEvent(new CustomMessageWrapper(new DummyCustomDiscoveryMessage(id))); } + /** + * Discovery SPI, that makes a thread to die when {@code blockMsgs} is set to {@code true}. + */ + private class DyingThreadDiscoverySpi extends TcpDiscoverySpi { + /** {@inheritDoc} */ + @Override protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) { + if (blockMsgs) + throw new RuntimeException("Thread is dying"); + } + } + /** * Discovery SPI, that makes a node stop sending messages when {@code blockMsgs} is set to {@code true}. */ From b6345031d3e6a7d7e8f81da86659f1aaa07442b0 Mon Sep 17 00:00:00 2001 From: Oleg Ignatenko Date: Thu, 22 Nov 2018 18:43:58 +0300 Subject: [PATCH 225/403] IGNITE-10174 Migrate examples tests from JUnit 3 to JUnit 4 - Fixes #5369. Signed-off-by: Pavel Kovalenko (cherry picked from commit b3a96454221d2e98acf0ebb9660f921b135ac338) --- .../examples/BasicExamplesSelfTest.java | 6 + .../CacheClientBinaryExampleTest.java | 3 + .../CacheContinuousQueryExamplesSelfTest.java | 4 + .../examples/CacheExamplesSelfTest.java | 4 + .../ComputeClientBinaryExampleTest.java | 2 + .../EncryptedCacheExampleSelfTest.java | 3 + .../examples/EventsExamplesSelfTest.java | 2 + .../examples/MessagingExamplesSelfTest.java | 3 + .../examples/SpringDataExampleSelfTest.java | 2 + .../ignite/examples/SqlExamplesSelfTest.java | 4 + .../IgniteExamplesJ8SelfTestSuite.java | 53 +++---- .../IgniteExamplesSelfTestSuite.java | 59 ++++---- .../junits/GridAbstractTest.java | 132 +++++++++++++----- .../common/GridAbstractExamplesTest.java | 5 +- .../junits/common/GridCommonAbstractTest.java | 4 +- .../junits/spi/GridSpiAbstractTest.java | 6 +- .../util/mbeans/GridMBeanDisableSelfTest.java | 4 +- 17 files changed, 196 insertions(+), 100 deletions(-) diff --git a/examples/src/test/java/org/apache/ignite/examples/BasicExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/BasicExamplesSelfTest.java index 41ae90a634449..fa1e630230774 100644 --- a/examples/src/test/java/org/apache/ignite/examples/BasicExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/BasicExamplesSelfTest.java @@ -23,6 +23,7 @@ import org.apache.ignite.examples.computegrid.ComputeRunnableExample; import org.apache.ignite.examples.datastructures.IgniteExecutorServiceExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * Closure examples self test. @@ -31,6 +32,7 @@ public class BasicExamplesSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testBroadcastExample() throws Exception { ComputeBroadcastExample.main(EMPTY_ARGS); } @@ -38,6 +40,7 @@ public void testBroadcastExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testCallableExample() throws Exception { ComputeCallableExample.main(EMPTY_ARGS); } @@ -45,6 +48,7 @@ public void testCallableExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testClosureExample() throws Exception { ComputeClosureExample.main(EMPTY_ARGS); } @@ -52,6 +56,7 @@ public void testClosureExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testExecutorExample() throws Exception { IgniteExecutorServiceExample.main(EMPTY_ARGS); } @@ -67,6 +72,7 @@ public void testExecutorExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testRunnableExample() throws Exception { ComputeRunnableExample.main(EMPTY_ARGS); } diff --git a/examples/src/test/java/org/apache/ignite/examples/CacheClientBinaryExampleTest.java b/examples/src/test/java/org/apache/ignite/examples/CacheClientBinaryExampleTest.java index 01be0bc7e0552..db25c603fd4fb 100644 --- a/examples/src/test/java/org/apache/ignite/examples/CacheClientBinaryExampleTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/CacheClientBinaryExampleTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.examples.binary.datagrid.CacheClientBinaryPutGetExample; import org.apache.ignite.examples.binary.datagrid.CacheClientBinaryQueryExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * @@ -33,6 +34,7 @@ public class CacheClientBinaryExampleTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testBinaryPutGetExample() throws Exception { CacheClientBinaryPutGetExample.main(new String[] {}); } @@ -40,6 +42,7 @@ public void testBinaryPutGetExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testBinaryQueryExample() throws Exception { CacheClientBinaryQueryExample.main(new String[] {}); } diff --git a/examples/src/test/java/org/apache/ignite/examples/CacheContinuousQueryExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/CacheContinuousQueryExamplesSelfTest.java index 1a1ae4e8e8e9e..d3aa704edcd2d 100644 --- a/examples/src/test/java/org/apache/ignite/examples/CacheContinuousQueryExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/CacheContinuousQueryExamplesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.examples.datagrid.CacheContinuousQueryExample; import org.apache.ignite.examples.datagrid.CacheContinuousQueryWithTransformerExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** */ @@ -28,6 +29,7 @@ public class CacheContinuousQueryExamplesSelfTest extends GridAbstractExamplesTe /** * @throws Exception If failed. */ + @Test public void testCacheContinuousAsyncQueryExample() throws Exception { CacheContinuousAsyncQueryExample.main(new String[] {}); } @@ -35,6 +37,7 @@ public void testCacheContinuousAsyncQueryExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testCacheContinuousQueryExample() throws Exception { CacheContinuousQueryExample.main(new String[] {}); } @@ -42,6 +45,7 @@ public void testCacheContinuousQueryExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testCacheContinuousQueryWithTransformerExample() throws Exception { CacheContinuousQueryWithTransformerExample.main(new String[] {}); } diff --git a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java index 0085573133ce8..258adbc5139eb 100644 --- a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.examples.datagrid.CacheEntryProcessorExample; import org.apache.ignite.examples.datagrid.CacheApiExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; //import org.apache.ignite.examples.datagrid.starschema.*; //import org.apache.ignite.examples.datagrid.store.dummy.*; @@ -33,6 +34,7 @@ public class CacheExamplesSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testCacheAffinityExample() throws Exception { CacheAffinityExample.main(EMPTY_ARGS); } @@ -40,6 +42,7 @@ public void testCacheAffinityExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testCacheEntryProcessorExample() throws Exception { CacheEntryProcessorExample.main(EMPTY_ARGS); } @@ -112,6 +115,7 @@ public void testCacheEntryProcessorExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testCacheApiExample() throws Exception { CacheApiExample.main(EMPTY_ARGS); } diff --git a/examples/src/test/java/org/apache/ignite/examples/ComputeClientBinaryExampleTest.java b/examples/src/test/java/org/apache/ignite/examples/ComputeClientBinaryExampleTest.java index 5dcad62d2ddb8..30480f0ea3a34 100644 --- a/examples/src/test/java/org/apache/ignite/examples/ComputeClientBinaryExampleTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/ComputeClientBinaryExampleTest.java @@ -18,6 +18,7 @@ import org.apache.ignite.examples.binary.computegrid.ComputeClientBinaryTaskExecutionExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * @@ -31,6 +32,7 @@ public class ComputeClientBinaryExampleTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testBinaryTaskExecutionExample() throws Exception { ComputeClientBinaryTaskExecutionExample.main(new String[] {}); } diff --git a/examples/src/test/java/org/apache/ignite/examples/EncryptedCacheExampleSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/EncryptedCacheExampleSelfTest.java index 234a3aca08d1c..66cd4cf341ac9 100644 --- a/examples/src/test/java/org/apache/ignite/examples/EncryptedCacheExampleSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/EncryptedCacheExampleSelfTest.java @@ -19,10 +19,12 @@ import org.apache.ignite.examples.encryption.EncryptedCacheExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** */ public class EncryptedCacheExampleSelfTest extends GridAbstractExamplesTest { + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { cleanPersistenceDir(); } @@ -30,6 +32,7 @@ public class EncryptedCacheExampleSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testBinaryPutGetExample() throws Exception { EncryptedCacheExample.main(new String[] {}); } diff --git a/examples/src/test/java/org/apache/ignite/examples/EventsExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/EventsExamplesSelfTest.java index 8e675a3c0be7b..635b247ad76f5 100644 --- a/examples/src/test/java/org/apache/ignite/examples/EventsExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/EventsExamplesSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.examples.events.EventsExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * Events examples self test. @@ -27,6 +28,7 @@ public class EventsExamplesSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testEventsExample() throws Exception { EventsExample.main(EMPTY_ARGS); } diff --git a/examples/src/test/java/org/apache/ignite/examples/MessagingExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/MessagingExamplesSelfTest.java index 3c94d3b20a6c2..0686dffc9c541 100644 --- a/examples/src/test/java/org/apache/ignite/examples/MessagingExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/MessagingExamplesSelfTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.examples.messaging.MessagingExample; import org.apache.ignite.examples.messaging.MessagingPingPongExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * Messaging examples self test. @@ -33,6 +34,7 @@ public class MessagingExamplesSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testMessagingExample() throws Exception { MessagingExample.main(EMPTY_ARGS); } @@ -40,6 +42,7 @@ public void testMessagingExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testMessagingPingPongExample() throws Exception { MessagingPingPongExample.main(EMPTY_ARGS); } diff --git a/examples/src/test/java/org/apache/ignite/examples/SpringDataExampleSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/SpringDataExampleSelfTest.java index 516ad4546347c..bba21b3d66808 100644 --- a/examples/src/test/java/org/apache/ignite/examples/SpringDataExampleSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/SpringDataExampleSelfTest.java @@ -18,6 +18,7 @@ import org.apache.ignite.examples.springdata.SpringDataExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * Spring Data example test. @@ -26,6 +27,7 @@ public class SpringDataExampleSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testSpringDataExample() throws Exception { SpringDataExample.main(EMPTY_ARGS); } diff --git a/examples/src/test/java/org/apache/ignite/examples/SqlExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/SqlExamplesSelfTest.java index 0bf01d8a534d2..c105335b9e73c 100644 --- a/examples/src/test/java/org/apache/ignite/examples/SqlExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/SqlExamplesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.examples.sql.SqlDmlExample; import org.apache.ignite.examples.sql.SqlQueriesExample; import org.apache.ignite.testframework.junits.common.GridAbstractExamplesTest; +import org.junit.Test; /** * SQL examples self test. @@ -29,6 +30,7 @@ public class SqlExamplesSelfTest extends GridAbstractExamplesTest { /** * @throws Exception If failed. */ + @Test public void testSqlJavaExample() throws Exception { SqlQueriesExample.main(EMPTY_ARGS); } @@ -36,6 +38,7 @@ public void testSqlJavaExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testSqlDmlExample() throws Exception { SqlDmlExample.main(EMPTY_ARGS); } @@ -43,6 +46,7 @@ public void testSqlDmlExample() throws Exception { /** * @throws Exception If failed. */ + @Test public void testSqlDdlExample() throws Exception { SqlDdlExample.main(EMPTY_ARGS); } diff --git a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesJ8SelfTestSuite.java b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesJ8SelfTestSuite.java index f73d977916845..f281ec57252bf 100644 --- a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesJ8SelfTestSuite.java +++ b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesJ8SelfTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import junit.framework.JUnit4TestAdapter; import junit.framework.TestSuite; import org.apache.ignite.examples.BasicExamplesMultiNodeSelfTest; import org.apache.ignite.examples.BasicExamplesSelfTest; @@ -45,35 +46,35 @@ public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("Ignite Examples Test Suite"); - suite.addTest(new TestSuite(CacheExamplesSelfTest.class)); - suite.addTest(new TestSuite(BasicExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(CacheExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(BasicExamplesSelfTest.class)); -// suite.addTest(new TestSuite(ContinuationExamplesSelfTest.class)); -// suite.addTest(new TestSuite(ContinuousMapperExamplesSelfTest.class)); -// suite.addTest(new TestSuite(DeploymentExamplesSelfTest.class)); - suite.addTest(new TestSuite(EventsExamplesSelfTest.class)); -// suite.addTest(new TestSuite(LifecycleExamplesSelfTest.class)); - suite.addTest(new TestSuite(MessagingExamplesSelfTest.class)); -// suite.addTest(new TestSuite(MemcacheRestExamplesSelfTest.class)); -// suite.addTest(new TestSuite(MonteCarloExamplesSelfTest.class)); -// suite.addTest(new TestSuite(TaskExamplesSelfTest.class)); -// suite.addTest(new TestSuite(SpringBeanExamplesSelfTest.class)); -// suite.addTest(new TestSuite(IgfsExamplesSelfTest.class)); -// suite.addTest(new TestSuite(CheckpointExamplesSelfTest.class)); -// suite.addTest(new TestSuite(HibernateL2CacheExampleSelfTest.class)); -// suite.addTest(new TestSuite(ClusterGroupExampleSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(ContinuationExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(ContinuousMapperExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(DeploymentExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(EventsExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(LifecycleExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(MessagingExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(MemcacheRestExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(MonteCarloExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(TaskExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(SpringBeanExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(IgfsExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(CheckpointExamplesSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(HibernateL2CacheExampleSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(ClusterGroupExampleSelfTest.class)); // Multi-node. - suite.addTest(new TestSuite(CacheExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(BasicExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(ContinuationExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(ContinuousMapperExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(DeploymentExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(EventsExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(TaskExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(MemcacheRestExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(MonteCarloExamplesMultiNodeSelfTest.class)); -// suite.addTest(new TestSuite(HibernateL2CacheExampleMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(CacheExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(BasicExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(ContinuationExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(ContinuousMapperExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(DeploymentExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(EventsExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(TaskExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(MemcacheRestExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(MonteCarloExamplesMultiNodeSelfTest.class)); +// suite.addTest(new JUnit4TestAdapter(HibernateL2CacheExampleMultiNodeSelfTest.class)); return suite; } diff --git a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java index 9553103c5d036..1c93a4bf37232 100644 --- a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java +++ b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import junit.framework.JUnit4TestAdapter; import junit.framework.TestSuite; import org.apache.ignite.examples.BasicExamplesMultiNodeSelfTest; import org.apache.ignite.examples.BasicExamplesSelfTest; @@ -65,39 +66,39 @@ public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("Ignite Examples Test Suite"); - suite.addTest(new TestSuite(CacheExamplesSelfTest.class)); - suite.addTest(new TestSuite(SqlExamplesSelfTest.class)); - suite.addTest(new TestSuite(BasicExamplesSelfTest.class)); - suite.addTest(new TestSuite(ContinuationExamplesSelfTest.class)); - suite.addTest(new TestSuite(ContinuousMapperExamplesSelfTest.class)); - suite.addTest(new TestSuite(DeploymentExamplesSelfTest.class)); - suite.addTest(new TestSuite(EventsExamplesSelfTest.class)); - suite.addTest(new TestSuite(LifecycleExamplesSelfTest.class)); - suite.addTest(new TestSuite(MessagingExamplesSelfTest.class)); - suite.addTest(new TestSuite(MemcacheRestExamplesSelfTest.class)); - suite.addTest(new TestSuite(MonteCarloExamplesSelfTest.class)); - suite.addTest(new TestSuite(TaskExamplesSelfTest.class)); - suite.addTest(new TestSuite(SpringBeanExamplesSelfTest.class)); - suite.addTest(new TestSuite(SpringDataExampleSelfTest.class)); - suite.addTest(new TestSuite(IgfsExamplesSelfTest.class)); - suite.addTest(new TestSuite(CheckpointExamplesSelfTest.class)); - suite.addTest(new TestSuite(ClusterGroupExampleSelfTest.class)); - suite.addTest(new TestSuite(CacheContinuousQueryExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(CacheExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(SqlExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(BasicExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(ContinuationExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(ContinuousMapperExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(DeploymentExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(EventsExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(LifecycleExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(MessagingExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(MemcacheRestExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(MonteCarloExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(TaskExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(SpringBeanExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(SpringDataExampleSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(IgfsExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(CheckpointExamplesSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(ClusterGroupExampleSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(CacheContinuousQueryExamplesSelfTest.class)); // Multi-node. - suite.addTest(new TestSuite(CacheExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(BasicExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(ContinuationExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(ContinuousMapperExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(DeploymentExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(EventsExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(TaskExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(MemcacheRestExamplesMultiNodeSelfTest.class)); - suite.addTest(new TestSuite(MonteCarloExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(CacheExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(BasicExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(ContinuationExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(ContinuousMapperExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(DeploymentExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(EventsExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(TaskExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(MemcacheRestExamplesMultiNodeSelfTest.class)); + suite.addTest(new JUnit4TestAdapter(MonteCarloExamplesMultiNodeSelfTest.class)); // Binary. - suite.addTest(new TestSuite(CacheClientBinaryExampleTest.class)); - suite.addTest(new TestSuite(ComputeClientBinaryExampleTest.class)); + suite.addTest(new JUnit4TestAdapter(CacheClientBinaryExampleTest.class)); + suite.addTest(new JUnit4TestAdapter(ComputeClientBinaryExampleTest.class)); // ML Grid. suite.addTest(IgniteExamplesMLTestSuite.suite()); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index 0be0d252b07eb..fb8dffa31880f 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -19,6 +19,7 @@ import java.io.ObjectStreamException; import java.io.Serializable; +import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; @@ -119,6 +120,12 @@ import org.apache.log4j.Priority; import org.apache.log4j.RollingFileAppender; import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; +import org.junit.runners.model.Statement; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.support.FileSystemXmlApplicationContext; @@ -164,6 +171,16 @@ public abstract class GridAbstractTest extends TestCase { /** */ protected static final String DEFAULT_CACHE_NAME = "default"; + /** Supports obtaining test name for JUnit4 cases. */ + @Rule public transient TestName nameRule = new TestName(); + + /** Manages test execution and reporting. */ + @Rule public transient TestRule runRule = (base, description) -> new Statement() { + @Override public void evaluate() throws Throwable { + runTest(base); + } + }; + /** */ private transient boolean startGrid; @@ -244,6 +261,13 @@ assert isJunitFrameworkClass() : "GridAbstractTest class cannot be extended dire this.startGrid = startGrid; } + /** {@inheritDoc} */ + @Override public String getName() { + String junit3Name = super.getName(); + + return junit3Name != null ? junit3Name : nameRule.getMethodName(); + } + /** * @param cls Class to create. * @return Instance of class. @@ -389,8 +413,8 @@ protected void time(String name, Runnable r) { } /** - * Runs given code in multiple threads and synchronously waits for all threads to complete. - * If any thread failed, exception will be thrown out of this method. + * Runs given code in multiple threads and synchronously waits for all threads to complete. If any thread failed, + * exception will be thrown out of this method. * * @param r Runnable. * @param threadNum Thread number. @@ -491,7 +515,8 @@ protected IgniteInternalFuture multithreadedAsync(Callable c, int threadNu * @throws Exception If failed. * @return Future. */ - protected IgniteInternalFuture multithreadedAsync(Callable c, int threadNum, String threadName) throws Exception { + protected IgniteInternalFuture multithreadedAsync(Callable c, int threadNum, + String threadName) throws Exception { return GridTestUtils.runMultiThreadedAsync(c, threadNum, threadName); } @@ -530,8 +555,8 @@ protected void beforeTest() throws Exception { } /** - * Called after execution of every test method in class or - * if {@link #beforeTest()} failed without test method execution. + * Called after execution of every test method in class or if {@link #beforeTest()} failed without test method + * execution. * * @throws Exception If failed. */ @@ -561,7 +586,8 @@ protected void afterTestsStopped() throws Exception { } /** {@inheritDoc} */ - @Override protected void setUp() throws Exception { + @Before + @Override public void setUp() throws Exception { stopGridErr = false; clsLdr = Thread.currentThread().getContextClassLoader(); @@ -593,8 +619,8 @@ protected void afterTestsStopped() throws Exception { if (isFirstTest()) { info(">>> Starting test class: " + testClassDescription() + " <<<"); - if(isSafeTopology()) - assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution:" + G.allGrids(); + if (isSafeTopology()) + assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution:" + G.allGrids(); if (startGrid) { IgniteConfiguration cfg = optimize(getConfiguration()); @@ -660,6 +686,34 @@ protected String testClassDescription() { return GridTestUtils.fullSimpleName(getClass()); } + /** + * @return Current test method. + * @throws NoSuchMethodError If method wasn't found for some reason. + */ + @NotNull protected Method currentTestMethod() { + if (currTestMtd == null) { + try { + currTestMtd = getClass().getMethod(getName()); + } + catch (NoSuchMethodException e) { + throw new NoSuchMethodError("Current test method is not found: " + getName()); + } + } + + return currTestMtd; + } + + /** + * Search for the annotation of the given type in current test method. + * + * @param annotationCls Type of annotation to look for. + * @param Annotation type. + * @return Instance of annotation if it is present in test method. + */ + @Nullable protected A currentTestAnnotation(Class annotationCls) { + return currentTestMethod().getAnnotation(annotationCls); + } + /** * @return Started grid. * @throws Exception If anything failed. @@ -946,7 +1000,7 @@ protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration */ protected Ignite startGridWithSpringCtx(String gridName, boolean client, String cfgUrl) throws Exception { IgniteBiTuple, ? extends GridSpringResourceContext> cfgMap = - IgnitionEx.loadConfigurations(cfgUrl); + IgnitionEx.loadConfigurations(cfgUrl); IgniteConfiguration cfg = F.first(cfgMap.get1()); @@ -1027,7 +1081,7 @@ protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration m.setAccessible(true); - cfg.setDiscoverySpi((DiscoverySpi) m.invoke(discoverySpi)); + cfg.setDiscoverySpi((DiscoverySpi)m.invoke(discoverySpi)); resetDiscovery = false; } @@ -1496,6 +1550,7 @@ protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws /** * Create instance of {@link BinaryMarshaller} suitable for use * without starting a grid upon an empty {@link IgniteConfiguration}. + * * @return Binary marshaller. * @throws IgniteCheckedException if failed. */ @@ -1506,6 +1561,7 @@ protected BinaryMarshaller createStandaloneBinaryMarshaller() throws IgniteCheck /** * Create instance of {@link BinaryMarshaller} suitable for use * without starting a grid upon given {@link IgniteConfiguration}. + * * @return Binary marshaller. * @throws IgniteCheckedException if failed. */ @@ -1593,10 +1649,10 @@ protected String home() throws IgniteCheckedException { /** * This method should be overridden by subclasses to change configuration parameters. * - * @return Grid configuration used for starting of grid. * @param igniteInstanceName Ignite instance name. * @param rsrcs Resources. * @throws Exception If failed. + * @return Grid configuration used for starting of grid. */ @SuppressWarnings("deprecation") protected IgniteConfiguration getConfiguration(String igniteInstanceName, IgniteTestResources rsrcs) @@ -1713,7 +1769,8 @@ protected static ClassLoader getExternalClassLoader() { } /** {@inheritDoc} */ - @Override protected void tearDown() throws Exception { + @After + @Override public void tearDown() throws Exception { long dur = System.currentTimeMillis() - ts; info(">>> Stopping test: " + testDescription() + " in " + dur + " ms <<<"); @@ -1951,7 +2008,7 @@ public static R executeOnLocalOrRemoteJvm(Ignite ignite, final TestIgniteCal * @param cache Cache. * @param job Job. */ - public static R executeOnLocalOrRemoteJvm(IgniteCache cache, TestCacheCallable job) { + public static R executeOnLocalOrRemoteJvm(IgniteCache cache, TestCacheCallable job) { Ignite ignite = cache.unwrap(Ignite.class); if (!isMultiJvmObject(ignite)) @@ -2010,7 +2067,7 @@ public static R executeRemotely(IgniteCacheProcessProxy cache, @Override public R call() throws Exception { Ignite ignite = Ignition.ignite(id); - IgniteCache cache = ignite.cache(cacheName); + IgniteCache cache = ignite.cache(cacheName); return job.call(ignite, cache); } @@ -2041,12 +2098,24 @@ protected synchronized TestCounters getTestCounters(IgniteConfiguration cfg) thr /** {@inheritDoc} */ @SuppressWarnings({"ProhibitedExceptionDeclared"}) @Override protected void runTest() throws Throwable { + runTest(new Statement() { + @Override public void evaluate() throws Throwable { + GridAbstractTest.super.runTest(); + } + }); + } + + /** */ + private void runTest(Statement testRoutine) throws Throwable { final AtomicReference ex = new AtomicReference<>(); Thread runner = new IgniteThread(getTestIgniteInstanceName(), "test-runner", new Runnable() { @Override public void run() { try { - runTestInternal(); + if (forceFailure) + fail("Forced failure: " + forceFailureMsg); + + testRoutine.evaluate(); } catch (Throwable e) { IgniteClosure hnd = errorHandler(); @@ -2063,7 +2132,7 @@ protected synchronized TestCounters getTestCounters(IgniteConfiguration cfg) thr if (runner.isAlive()) { U.error(log, "Test has been timed out and will be interrupted (threads dump will be taken before interruption) [" + - "test=" + getName() + ", timeout=" + getTestTimeout() + ']'); + "test=" + getName() + ", timeout=" + getTestTimeout() + ']'); List nodes = IgnitionEx.allGridsx(); @@ -2081,7 +2150,7 @@ protected synchronized TestCounters getTestCounters(IgniteConfiguration cfg) thr U.join(runner, log); throw new TimeoutException("Test has been timed out [test=" + getName() + ", timeout=" + - getTestTimeout() + ']' ); + getTestTimeout() + ']'); } Throwable t = ex.get(); @@ -2096,8 +2165,7 @@ protected synchronized TestCounters getTestCounters(IgniteConfiguration cfg) thr } /** - * @return Error handler to process all uncaught exceptions of the test run - * ({@code null} by default). + * @return Error handler to process all uncaught exceptions of the test run ({@code null} by default). */ protected IgniteClosure errorHandler() { return null; @@ -2123,17 +2191,6 @@ public void forceTestCount(int cnt) { forceTestCnt = true; } - /** - * @throws Throwable If failed. - */ - @SuppressWarnings({"ProhibitedExceptionDeclared"}) - private void runTestInternal() throws Throwable { - if (forceFailure) - fail("Forced failure: " + forceFailureMsg); - else - super.runTest(); - } - /** * @return Test case timeout. */ @@ -2156,7 +2213,7 @@ private long getDefaultTestTimeout() { /** * @param store Store. */ - protected Factory singletonFactory(T store) { + protected static Factory singletonFactory(T store) { return notSerializableProxy(new FactoryBuilder.SingletonFactory<>(store), Factory.class); } @@ -2164,7 +2221,7 @@ protected Factory singletonFactory(T store) { * @param obj Object that should be wrap proxy * @return Created proxy. */ - protected T notSerializableProxy(final T obj) { + protected static T notSerializableProxy(final T obj) { Class cls = (Class)obj.getClass(); Class[] interfaces = (Class[])cls.getInterfaces(); @@ -2182,7 +2239,7 @@ protected T notSerializableProxy(final T obj) { * @param itfClses Interfaces that should be implemented by proxy (vararg parameter) * @return Created proxy. */ - protected T notSerializableProxy(final T obj, Class itfCls, Class ... itfClses) { + protected static T notSerializableProxy(final T obj, Class itfCls, Class... itfClses) { Class[] itfs = Arrays.copyOf(itfClses, itfClses.length + 3); itfs[itfClses.length] = itfCls; @@ -2205,7 +2262,7 @@ protected T notSerializableProxy(final T obj, Class itfCls, Class * @param obj Object that must not be changed after serialization/deserialization. * @return An object to return from writeReplace() */ - private Object supressSerialization(Object obj) { + private static Object supressSerialization(Object obj) { SerializableProxy res = new SerializableProxy(UUID.randomUUID()); serializedObj.put(res.uuid, obj); @@ -2243,7 +2300,7 @@ private void awaitTopologyChange() throws IgniteInterruptedCheckedException { AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion(); if (!topVer.equals(exchVer)) { - info("Topology version mismatch [node=" + g.name() + + info("Topology version mismatch [node=" + g.name() + ", exchVer=" + exchVer + ", topVer=" + topVer + ']'); @@ -2258,6 +2315,7 @@ private void awaitTopologyChange() throws IgniteInterruptedCheckedException { } } } + /** * @param expSize Expected nodes number. * @throws Exception If failed. @@ -2273,7 +2331,7 @@ protected void waitForTopology(final int expSize) throws Exception { return false; } - for (Ignite node: nodes) { + for (Ignite node : nodes) { try { IgniteFuture reconnectFut = node.cluster().clientReconnectFuture(); @@ -2320,7 +2378,7 @@ public static void doSleep(long millis) { * @param cacheName Cache name. * @return Cache group ID for given cache name. */ - protected final int groupIdForCache(Ignite node, String cacheName) { + protected static final int groupIdForCache(Ignite node, String cacheName) { for (CacheGroupContext grp : ((IgniteKernal)node).context().cache().cacheGroups()) { if (grp.hasCache(cacheName)) return grp.groupId(); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridAbstractExamplesTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridAbstractExamplesTest.java index 1e6c84ee1f27a..70fd33035cdb1 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridAbstractExamplesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridAbstractExamplesTest.java @@ -20,10 +20,13 @@ import java.io.FileReader; import java.util.Properties; import org.apache.ignite.internal.util.typedef.internal.U; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; /** * Base class for examples test. */ +@RunWith(JUnit4.class) public abstract class GridAbstractExamplesTest extends GridCommonAbstractTest { /** */ protected static final String[] EMPTY_ARGS = new String[0]; @@ -75,4 +78,4 @@ protected final void startRemoteNodes() throws Exception { protected String defaultConfig() { return DFLT_CFG; } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 7091a0990d71c..43a9e5c817835 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -481,7 +481,7 @@ protected GridNearCacheAdapter near(int idx, String cache) { } /** {@inheritDoc} */ - @Override protected void setUp() throws Exception { + @Override public void setUp() throws Exception { // Disable SSL hostname verifier. HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { @Override public boolean verify(String s, SSLSession sslSes) { @@ -495,7 +495,7 @@ protected GridNearCacheAdapter near(int idx, String cache) { } /** {@inheritDoc} */ - @Override protected void tearDown() throws Exception { + @Override public void tearDown() throws Exception { getTestCounters().incrementStopped(); super.tearDown(); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java index 101d0163d469f..426c5816e7af6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/spi/GridSpiAbstractTest.java @@ -126,7 +126,7 @@ private void resetTestData() throws Exception { /** * @throws Exception If failed. */ - @Override protected final void setUp() throws Exception { + @Override public final void setUp() throws Exception { // Need to change classloader here, although it also handled in the parent class // the current test initialisation procedure doesn't allow us to setUp the parent first. cl = Thread.currentThread().getContextClassLoader(); @@ -490,7 +490,7 @@ protected UUID getNodeId() throws Exception { /** * @throws Exception If failed. */ - @Override protected final void tearDown() throws Exception { + @Override public final void tearDown() throws Exception { getTestCounters().incrementStopped(); boolean wasLast = isLastTest(); @@ -730,4 +730,4 @@ private static class SecurityPermissionSetImpl implements SecurityPermissionSet return null; } } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java index b98cd46c369fe..356c05a2700d1 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java @@ -33,14 +33,14 @@ */ public class GridMBeanDisableSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ - @Override protected void setUp() throws Exception { + @Override public void setUp() throws Exception { IgniteUtils.IGNITE_MBEANS_DISABLED = true; super.setUp(); } /** {@inheritDoc} */ - @Override protected void tearDown() throws Exception { + @Override public void tearDown() throws Exception { IgniteUtils.IGNITE_MBEANS_DISABLED = false; super.tearDown(); From fc199dd984d8922bf2410ebe3c096ed0268e3760 Mon Sep 17 00:00:00 2001 From: Oleg Ignatenko Date: Tue, 11 Dec 2018 19:09:02 +0300 Subject: [PATCH 226/403] IGNITE-10174 merge error corrected -- verified with diffs overview and rebuild --- .../org/apache/ignite/testframework/junits/GridAbstractTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index fb8dffa31880f..b2d12fda16999 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -119,6 +119,7 @@ import org.apache.log4j.PatternLayout; import org.apache.log4j.Priority; import org.apache.log4j.RollingFileAppender; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; From 977ba40cd7d8f01817a6f49f8cb89066a5dee5bf Mon Sep 17 00:00:00 2001 From: zstan Date: Tue, 11 Dec 2018 19:06:10 +0300 Subject: [PATCH 227/403] IGNITE-10492 Do not normalize QueryEntities if cache is in restarting state - Fixes #5537. Signed-off-by: Alexey Goncharuk (cherry picked from commit 47b1be6) --- .../delta/MetaPageUpdateNextSnapshotId.java | 10 +++++----- .../processors/cache/ClusterCachesInfo.java | 2 +- .../processors/cache/GridCacheProcessor.java | 12 ++++++++---- .../persistence/GridCacheOffheapManager.java | 18 +++--------------- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java index 2046ecd274d95..5068fe5b45ec5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java @@ -27,22 +27,22 @@ */ public class MetaPageUpdateNextSnapshotId extends PageDeltaRecord { /** */ - private final long nextSnapshotId; + private final long nextSnapshotTag; /** * @param pageId Meta page ID. */ - public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotId) { + public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotTag) { super(grpId, pageId); - this.nextSnapshotId = nextSnapshotId; + this.nextSnapshotTag = nextSnapshotTag; } /** {@inheritDoc} */ @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException { PageMetaIO io = PageMetaIO.VERSIONS.forPage(pageAddr); - io.setNextSnapshotTag(pageAddr, nextSnapshotId); + io.setNextSnapshotTag(pageAddr, nextSnapshotTag); } /** {@inheritDoc} */ @@ -54,7 +54,7 @@ public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotId) * @return Root ID. */ public long nextSnapshotId() { - return nextSnapshotId; + return nextSnapshotTag; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 6718cf3ef7341..b6b6097ec6e3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -95,7 +95,7 @@ class ClusterCachesInfo { private final ConcurrentMap registeredTemplates = new ConcurrentHashMap<>(); /** Caches currently being restarted. */ - private final Collection restartingCaches = new GridConcurrentHashSet<>(); + private final Set restartingCaches = new GridConcurrentHashSet<>(); /** */ private final IgniteLogger log; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 30a2a14f8ff8c..81285c8917826 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -5315,17 +5315,21 @@ else if (!isLocalAffinity(descCfg)) } } else { + CacheConfiguration cfg = new CacheConfiguration(ccfg); + req.deploymentId(IgniteUuid.randomUuid()); - CacheConfiguration cfg = new CacheConfiguration(ccfg); + req.startCacheConfiguration(cfg); CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(cfg); initialize(cfg, cacheObjCtx); - req.startCacheConfiguration(cfg); - req.schema(new QuerySchema(qryEntities != null ? QueryUtils.normalizeQueryEntities(qryEntities, cfg) - : cfg.getQueryEntities())); + if (cachesInfo.restartingCaches().contains(req.cacheName())) + req.schema(new QuerySchema(qryEntities)); + else + req.schema(new QuerySchema(qryEntities != null ? QueryUtils.normalizeQueryEntities(qryEntities, cfg) + : cfg.getQueryEntities())); } } else { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index e1bd8dd77db35..04255c01e08e2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -179,17 +179,7 @@ public IndexStorage getIndexStorage() { boolean needSnapshot = ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName()); - boolean hasNonEmptyGroups = false; - - for (CacheDataStore store : partDataStores.values()) { - if (notEmpty(store)) { - hasNonEmptyGroups = true; - - break; - } - } - - if (needSnapshot && hasNonEmptyGroups) { + if (needSnapshot) { if (execSvc == null) updateSnapshotTag(ctx); else { @@ -703,10 +693,8 @@ private void updateSnapshotTag(Context ctx) throws IgniteCheckedException { log.debug("Save next snapshot before checkpoint start for grId = " + grpId + ", nextSnapshotTag = " + nextSnapshotTag); - if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId, - metaPage, wal, null)) - wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId, - nextSnapshotTag + 1)); + if (!wal.disabled(grpId)) + wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId, nextSnapshotTag + 1)); addPartition( null, From d338e49856867d034c791dfa81fec8549e5f551d Mon Sep 17 00:00:00 2001 From: deviljelly Date: Fri, 30 Nov 2018 16:40:40 +0300 Subject: [PATCH 228/403] IGNITE-9298 SSL support in control.sh - Fixes #5516. Fixes #4684. Fixes #4697. Signed-off-by: Ilya Kasnacheev Co-authored-by: a-polyakov Co-authored-by: Ilya Kasnacheev (cherry picked from commit 6d7847c7f1427a337975d240b10bec8b10a8d3f2) --- .../internal/commandline/Arguments.java | 100 ++++++++++++- .../internal/commandline/CommandHandler.java | 139 +++++++++++++++++- .../CommandHandlerParsingTest.java | 39 +++++ .../IgniteBasicWithPersistenceTestSuite.java | 2 + .../util/GridCommandHandlerSslTest.java | 88 +++++++++++ 5 files changed, 363 insertions(+), 5 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java index 6367eef516101..07e7b104eed0d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java @@ -77,6 +77,30 @@ public class Arguments { /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */ private long pingInterval; + /** SSL Protocol. */ + private String sslProtocol; + + /** SSL Key Algorithm. */ + private String sslKeyAlgorithm; + + /** Keystore. */ + private String sslKeyStorePath; + + /** Keystore Type. */ + private String sslKeyStoreType; + + /** Keystore Password. */ + private char[] sslKeyStorePassword; + + /** Truststore. */ + private String sslTrustStorePath; + + /** Truststore Type. */ + private String sslTrustStoreType; + + /** Truststore Password. */ + private char[] sslTrustStorePassword; + /** * @param cmd Command. * @param host Host. @@ -92,10 +116,20 @@ public class Arguments { * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}. * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}. * @param autoConfirmation Auto confirmation flag. + * @param sslProtocol SSL Protocol. + * @param sslKeyAlgorithm SSL Key Algorithm. + * @param sslKeyStorePath Keystore. + * @param sslKeyStorePassword Keystore Password. + * @param sslKeyStoreType Keystore Type. + * @param sslTrustStorePath Truststore. + * @param sslTrustStorePassword Truststore Password. + * @param sslTrustStoreType Truststore Type. */ public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct, String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs, - Long pingTimeout, Long pingInterval, boolean autoConfirmation) { + Long pingTimeout, Long pingInterval, boolean autoConfirmation, String sslProtocol, String sslKeyAlgorithm, + String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType, + String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType) { this.cmd = cmd; this.host = host; this.port = port; @@ -110,6 +144,14 @@ public Arguments(Command cmd, String host, String port, String user, String pwd, this.pingTimeout = pingTimeout; this.pingInterval = pingInterval; this.autoConfirmation = autoConfirmation; + this.sslProtocol = sslProtocol; + this.sslKeyAlgorithm = sslKeyAlgorithm; + this.sslKeyStorePath = sslKeyStorePath; + this.sslKeyStoreType = sslKeyStoreType; + this.sslKeyStorePassword = sslKeyStorePassword; + this.sslTrustStorePath = sslTrustStorePath; + this.sslTrustStoreType = sslTrustStoreType; + this.sslTrustStorePassword = sslTrustStorePassword; } /** @@ -227,4 +269,60 @@ public long pingInterval() { public boolean autoConfirmation() { return autoConfirmation; } + + /** + * @return SSL protocol + */ + public String sslProtocol() { + return sslProtocol; + } + + /** + * @return SSL Key Algorithm + */ + public String sslKeyAlgorithm() { + return sslKeyAlgorithm; + } + + /** + * @return Keystore + */ + public String sslKeyStorePath() { + return sslKeyStorePath; + } + + /** + * @return Keystore type + */ + public String sslKeyStoreType() { + return sslKeyStoreType; + } + + /** + * @return Keystore password + */ + public char[] sslKeyStorePassword() { + return sslKeyStorePassword; + } + + /** + * @return Truststore + */ + public String sslTrustStorePath() { + return sslTrustStorePath; + } + + /** + * @return Truststore type + */ + public String sslTrustStoreType() { + return sslTrustStoreType; + } + + /** + * @return Truststore password + */ + public char[] sslTrustStorePassword() { + return sslTrustStorePassword; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 899f7ef845a05..6d90d966cad61 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -54,6 +54,7 @@ import org.apache.ignite.internal.client.GridClientNode; import org.apache.ignite.internal.client.GridServerUnreachableException; import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException; +import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory; import org.apache.ignite.internal.commandline.cache.CacheArguments; import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTask; @@ -124,6 +125,7 @@ import org.apache.ignite.plugin.security.SecurityCredentials; import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider; import org.apache.ignite.plugin.security.SecurityCredentialsProvider; +import org.apache.ignite.ssl.SslContextFactory; import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND; import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR; @@ -196,6 +198,32 @@ public class CommandHandler { /** */ private static final String CMD_USER_ATTRIBUTES = "--user-attributes"; + // SSL configuration section + + /** */ + private static final String CMD_SSL_PROTOCOL = "--ssl-protocol"; + + /** */ + private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm"; + + /** */ + private static final String CMD_KEYSTORE = "--keystore"; + + /** */ + private static final String CMD_KEYSTORE_PASSWORD = "--keystore-password"; + + /** */ + private static final String CMD_KEYSTORE_TYPE = "--keystore-type"; + + /** */ + private static final String CMD_TRUSTSTORE = "--truststore"; + + /** */ + private static final String CMD_TRUSTSTORE_PASSWORD = "--truststore-password"; + + /** */ + private static final String CMD_TRUSTSTORE_TYPE = "--truststore-type"; + /** List of optional auxiliary commands. */ private static final Set AUX_COMMANDS = new HashSet<>(); @@ -208,6 +236,14 @@ public class CommandHandler { AUX_COMMANDS.add(CMD_AUTO_CONFIRMATION); AUX_COMMANDS.add(CMD_PING_INTERVAL); AUX_COMMANDS.add(CMD_PING_TIMEOUT); + AUX_COMMANDS.add(CMD_SSL_PROTOCOL); + AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM); + AUX_COMMANDS.add(CMD_KEYSTORE); + AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD); + AUX_COMMANDS.add(CMD_KEYSTORE_TYPE); + AUX_COMMANDS.add(CMD_TRUSTSTORE); + AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD); + AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE); } /** Broadcast uuid. */ @@ -316,7 +352,13 @@ public class CommandHandler { private static final String UTILITY_NAME = "control.sh"; /** Common options. */ - private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"), op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT")); + private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), + op(CMD_PORT, "PORT"), op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), + op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"), + op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL"), op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"), + op(CMD_KEYSTORE, "KEYSTORE"), op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"), + op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"), op(CMD_TRUSTSTORE, "TRUSTSTORE"), + op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"), op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD")); /** Utility name with common options. */ private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS); @@ -1777,6 +1819,22 @@ Arguments parseAndValidate(List rawArgs) { VisorTxTaskArg txArgs = null; + String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL; + + String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM; + + String sslKeyStorePath = null; + + String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE; + + char sslKeyStorePassword[] = null; + + String sslTrustStorePath = null; + + String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE; + + char sslTrustStorePassword[] = null; + while (hasNextArg()) { String str = nextArg("").toLowerCase(); @@ -1889,6 +1947,46 @@ Arguments parseAndValidate(List rawArgs) { break; + case CMD_SSL_PROTOCOL: + sslProtocol = nextArg("Expected SSL protocol"); + + break; + + case CMD_SSL_KEY_ALGORITHM: + sslKeyAlgorithm = nextArg("Expected SSL key algorithm"); + + break; + + case CMD_KEYSTORE: + sslKeyStorePath = nextArg("Expected keystore path"); + + break; + + case CMD_KEYSTORE_PASSWORD: + sslKeyStorePassword = nextArg("Expected keystore password").toCharArray(); + + break; + + case CMD_KEYSTORE_TYPE: + sslKeyStoreType = nextArg("Expected keystore type"); + + break; + + case CMD_TRUSTSTORE: + sslTrustStorePath = nextArg("Expected truststore path"); + + break; + + case CMD_TRUSTSTORE_PASSWORD: + sslTrustStorePassword = nextArg("Expected truststore password").toCharArray(); + + break; + + case CMD_TRUSTSTORE_TYPE: + sslTrustStoreType = nextArg("Expected truststore type"); + + break; + case CMD_AUTO_CONFIRMATION: autoConfirmation = true; @@ -1911,7 +2009,9 @@ Arguments parseAndValidate(List rawArgs) { Command cmd = commands.get(0); return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs, - pingTimeout, pingInterval, autoConfirmation); + pingTimeout, pingInterval, autoConfirmation, + sslProtocol, sslKeyAlgorithm, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType, + sslTrustStorePath, sslTrustStorePassword, sslTrustStoreType); } /** @@ -2436,6 +2536,10 @@ public int execute(List rawArgs) { log(i("PORT=" + DFLT_PORT, 2)); log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2)); log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2)); + log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2)); + log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2)); + log(i("KEYSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + log(i("TRUSTSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); nl(); log("Exit codes:"); @@ -2482,8 +2586,7 @@ public int execute(List rawArgs) { if (securityCredential == null) { securityCredential = new SecurityCredentialsBasicProvider( - new SecurityCredentials(args.getUserName(), args.getPassword()) - ); + new SecurityCredentials(args.getUserName(), args.getPassword())); clientCfg.setSecurityCredentialsProvider(securityCredential); } @@ -2492,6 +2595,34 @@ public int execute(List rawArgs) { credential.setPassword(args.getPassword()); } + if (!F.isEmpty(args.sslKeyStorePath())) { + GridSslBasicContextFactory factory = new GridSslBasicContextFactory(); + + factory.setProtocol(args.sslProtocol()); + + factory.setKeyAlgorithm(args.sslKeyAlgorithm()); + + factory.setKeyStoreFilePath(args.sslKeyStorePath()); + + if (args.sslKeyStorePassword() != null) + factory.setKeyStorePassword(args.sslKeyStorePassword()); + + factory.setKeyStoreType(args.sslKeyStoreType()); + + if (F.isEmpty(args.sslTrustStorePath())) + factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager()); + else { + factory.setTrustStoreFilePath(args.sslTrustStorePath()); + + if (args.sslTrustStorePassword() != null) + factory.setTrustStorePassword(args.sslTrustStorePassword()); + + factory.setTrustStoreType(args.sslTrustStoreType()); + } + + clientCfg.setSslContextFactory(factory); + } + try (GridClient client = GridClientFactory.start(clientCfg)) { switch (args.command()) { case ACTIVATE: diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 988b1d7e94a40..203d6d06d0ba3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -38,6 +38,7 @@ import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH; import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE; import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT; +import static org.junit.Assert.assertArrayEquals; /** * Tests Command Handler parsing arguments. @@ -166,6 +167,44 @@ public void testExperimentalCommandIsDisabled() { } } + /** + * Tests parsing and validation for the SSL arguments. + */ + public void testParseAndValidateSSLArguments() { + CommandHandler hnd = new CommandHandler(); + + for (Command cmd : Command.values()) { + + if (cmd == Command.CACHE || cmd == Command.WAL) + continue; // --cache subcommand requires its own specific arguments. + + try { + hnd.parseAndValidate(asList("--truststore")); + + fail("expected exception: Expected truststore"); + } + catch (IllegalArgumentException e) { + e.printStackTrace(); + } + + Arguments args = hnd.parseAndValidate(asList("--keystore", "testKeystore", "--keystore-password", "testKeystorePassword", "--keystore-type", "testKeystoreType", + "--truststore", "testTruststore", "--truststore-password", "testTruststorePassword", "--truststore-type", "testTruststoreType", + "--ssl-key-algorithm", "testSSLKeyAlgorithm", "--ssl-protocol", "testSSLProtocol", cmd.text())); + + assertEquals("testSSLProtocol", args.sslProtocol()); + assertEquals("testSSLKeyAlgorithm", args.sslKeyAlgorithm()); + assertEquals("testKeystore", args.sslKeyStorePath()); + assertArrayEquals("testKeystorePassword".toCharArray(), args.sslKeyStorePassword()); + assertEquals("testKeystoreType", args.sslKeyStoreType()); + assertEquals("testTruststore", args.sslTrustStorePath()); + assertArrayEquals("testTruststorePassword".toCharArray(), args.sslTrustStorePassword()); + assertEquals("testTruststoreType", args.sslTrustStoreType()); + + assertEquals(cmd, args.command()); + } + } + + /** * Tests parsing and validation for user and password arguments. */ diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java index 68085db0e25b7..36f77ccc96912 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest; import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest; import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest; +import org.apache.ignite.util.GridCommandHandlerSslTest; import org.apache.ignite.util.GridCommandHandlerTest; import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest; import org.jetbrains.annotations.Nullable; @@ -67,6 +68,7 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex suite.addTestSuite(FailureHandlingConfigurationTest.class); suite.addTestSuite(GridCommandHandlerTest.class); + suite.addTestSuite(GridCommandHandlerSslTest.class); suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class); suite.addTestSuite(GridNodeMetricsLogPdsSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java new file mode 100644 index 0000000000000..59155e5b3713a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.util; + +import java.util.Arrays; +import org.apache.ignite.Ignite; +import org.apache.ignite.configuration.ConnectorConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.commandline.CommandHandler; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_CONNECTION_FAILED; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; + +/** + * Command line handler test. + */ +public class GridCommandHandlerSslTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + cleanPersistenceDir(); + + stopAllGrids(); + + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration()); + cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().setMaxSize(100 * 1024 * 1024); + cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().setPersistenceEnabled(true); + + cfg.setConnectorConfiguration(new ConnectorConfiguration()); + cfg.getConnectorConfiguration().setSslEnabled(true); + cfg.setSslContextFactory(GridTestUtils.sslFactory()); + + return cfg; + } + + /** + * Test activation works via control.sh + * + * @throws Exception If failed. + */ + public void testActivate() throws Exception { + Ignite ignite = startGrids(1); + + assertFalse(ignite.cluster().active()); + + final CommandHandler cmd = new CommandHandler(); + assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList( + "--activate", + "--keystore", GridTestUtils.keyStorePath("node01"), + "--keystore-password", GridTestUtils.keyStorePassword()))); + + assertTrue(ignite.cluster().active()); + + assertEquals(EXIT_CODE_CONNECTION_FAILED, cmd.execute(Arrays.asList("--deactivate", "--yes"))); + } + +} From 7b3eec7f875464859c64f6be55cbf89f9621cd9e Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Tue, 11 Dec 2018 16:40:07 +0300 Subject: [PATCH 229/403] IGNITE-10300 Fixed incorrect error message after authentication retries - Fixes #5425. Signed-off-by: Alexey Goncharuk (cherry picked from commit ed9c1377c0d38bb1bb3352b2b943e69e776d917c) --- .../apache/ignite/internal/commandline/CommandHandler.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 6d90d966cad61..f6094c7527298 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -2691,8 +2691,13 @@ public int execute(List rawArgs) { tryConnectMaxCount--; } - else + else { + if (tryConnectMaxCount == 0) + throw new GridClientAuthenticationException("Authentication error, maximum number of " + + "retries exceeded"); + throw e; + } } } return EXIT_CODE_OK; From 49832c8289eabb0d12e27b273467f9ca975fb4fc Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 12 Dec 2018 18:32:28 +0300 Subject: [PATCH 230/403] IGNITE-10556 Skip WAL records deserialization on recovery if possible - Fixes #5624. Signed-off-by: Pavel Kovalenko (cherry picked from commit 27786a05e2993334c5db403ed59fdaa0c36b0722) Signed-off-by: Pavel Kovalenko --- .../encryption/example-encrypted-store.xml | 2 + .../wal/IgniteWriteAheadLogManager.java | 16 + .../internal/pagemem/wal/WALIterator.java | 6 +- .../pagemem/wal/record/DataRecord.java | 5 +- .../pagemem/wal/record/MvccTxRecord.java | 8 +- .../pagemem/wal/record/WALRecord.java | 167 +++++++--- .../GridCacheDatabaseSharedManager.java | 310 +++++++++++------- .../persistence/GridCacheOffheapManager.java | 11 + .../wal/AbstractWalRecordsIterator.java | 11 + .../wal/FileWriteAheadLogManager.java | 11 +- .../FsyncModeFileWriteAheadLogManager.java | 14 +- .../RecordSerializerFactoryImpl.java | 7 +- .../wal/serializer/RecordV1Serializer.java | 3 +- .../wal/serializer/RecordV2Serializer.java | 3 +- .../db/IgniteLogicalRecoveryTest.java | 17 +- .../persistence/pagemem/NoOpWALManager.java | 7 + 16 files changed, 411 insertions(+), 187 deletions(-) diff --git a/examples/config/encryption/example-encrypted-store.xml b/examples/config/encryption/example-encrypted-store.xml index e526ae3e7b401..7ce5482ad5ca7 100644 --- a/examples/config/encryption/example-encrypted-store.xml +++ b/examples/config/encryption/example-encrypted-store.xml @@ -28,6 +28,8 @@ + + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java index 679eec98acc01..8a4d3a706658e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java @@ -24,6 +24,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedManager; import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.jetbrains.annotations.Nullable; /** * @@ -101,6 +103,20 @@ public WALPointer log(WALRecord entry, RolloverType rolloverType) */ public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException; + /** + * Invoke this method to iterate over the written log entries. + * + * @param start Optional WAL pointer from which to start iteration. + * @param recordDeserializeFilter Specify a filter to skip WAL records. Those records will not be explicitly deserialized. + * @return Records iterator. + * @throws IgniteException If failed to start iteration. + * @throws StorageException If IO error occurred while reading WAL entries. + */ + public WALIterator replay( + WALPointer start, + @Nullable IgniteBiPredicate recordDeserializeFilter + ) throws IgniteCheckedException, StorageException; + /** * Invoke this method to reserve WAL history since provided pointer and prevent it's deletion. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/WALIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/WALIterator.java index 14fdfda8fe9cd..b3c9726f7f88c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/WALIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/WALIterator.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.pagemem.wal; +import java.util.Optional; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.util.lang.GridCloseableIterator; import org.apache.ignite.lang.IgniteBiTuple; @@ -25,5 +26,8 @@ * */ public interface WALIterator extends GridCloseableIterator> { - // Iterator alias. + /** + * @return Pointer of last read valid record. Empty if no records were read. + */ + public Optional lastRead(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java index d5ab53a7226c2..ef6c3bafbc307 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java @@ -78,9 +78,12 @@ public DataRecord(List writeEntries, long timestamp) { /** * @param writeEntries Write entries. + * @return {@code this} for chaining. */ - public void setWriteEntries(List writeEntries) { + public DataRecord setWriteEntries(List writeEntries) { this.writeEntries = writeEntries; + + return this; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MvccTxRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MvccTxRecord.java index 82c4409951f8d..86ad983db209e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MvccTxRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MvccTxRecord.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.Map; import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion; +import org.apache.ignite.internal.processors.cache.mvcc.txlog.TxLog; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.transactions.TransactionState; @@ -29,7 +30,7 @@ * Logical data record indented for MVCC transaction related actions.
* This record is marker of prepare, commit, and rollback transactions. */ -public class MvccTxRecord extends TxRecord { +public class MvccTxRecord extends TxRecord implements WalRecordCacheGroupAware { /** Transaction mvcc snapshot version. */ private final MvccVersion mvccVer; @@ -85,6 +86,11 @@ public MvccVersion mvccVersion() { return mvccVer; } + /** {@inheritDoc} */ + @Override public int groupId() { + return TxLog.TX_LOG_CACHE_ID; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(MvccTxRecord.class, this, "super", super.toString()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java index ad8a2a407c75c..5d7276838ed3d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java @@ -19,9 +19,12 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; +import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordPurpose.*; + /** * Log entry abstract class. */ @@ -32,126 +35,126 @@ public abstract class WALRecord { */ public enum RecordType { /** */ - TX_RECORD, + TX_RECORD (LOGICAL), /** */ - PAGE_RECORD, + PAGE_RECORD (PHYSICAL), /** */ - DATA_RECORD, + DATA_RECORD (LOGICAL), /** Checkpoint (begin) record */ - CHECKPOINT_RECORD, + CHECKPOINT_RECORD (PHYSICAL), /** WAL segment header record. */ - HEADER_RECORD, + HEADER_RECORD (INTERNAL), // Delta records. /** */ - INIT_NEW_PAGE_RECORD, + INIT_NEW_PAGE_RECORD (PHYSICAL), /** */ - DATA_PAGE_INSERT_RECORD, + DATA_PAGE_INSERT_RECORD (PHYSICAL), /** */ - DATA_PAGE_INSERT_FRAGMENT_RECORD, + DATA_PAGE_INSERT_FRAGMENT_RECORD (PHYSICAL), /** */ - DATA_PAGE_REMOVE_RECORD, + DATA_PAGE_REMOVE_RECORD (PHYSICAL), /** */ - DATA_PAGE_SET_FREE_LIST_PAGE, + DATA_PAGE_SET_FREE_LIST_PAGE (PHYSICAL), /** */ - BTREE_META_PAGE_INIT_ROOT, + BTREE_META_PAGE_INIT_ROOT (PHYSICAL), /** */ - BTREE_META_PAGE_ADD_ROOT, + BTREE_META_PAGE_ADD_ROOT (PHYSICAL), /** */ - BTREE_META_PAGE_CUT_ROOT, + BTREE_META_PAGE_CUT_ROOT (PHYSICAL), /** */ - BTREE_INIT_NEW_ROOT, + BTREE_INIT_NEW_ROOT (PHYSICAL), /** */ - BTREE_PAGE_RECYCLE, + BTREE_PAGE_RECYCLE (PHYSICAL), /** */ - BTREE_PAGE_INSERT, + BTREE_PAGE_INSERT (PHYSICAL), /** */ - BTREE_FIX_LEFTMOST_CHILD, + BTREE_FIX_LEFTMOST_CHILD (PHYSICAL), /** */ - BTREE_FIX_COUNT, + BTREE_FIX_COUNT (PHYSICAL), /** */ - BTREE_PAGE_REPLACE, + BTREE_PAGE_REPLACE (PHYSICAL), /** */ - BTREE_PAGE_REMOVE, + BTREE_PAGE_REMOVE (PHYSICAL), /** */ - BTREE_PAGE_INNER_REPLACE, + BTREE_PAGE_INNER_REPLACE (PHYSICAL), /** */ - BTREE_FIX_REMOVE_ID, + BTREE_FIX_REMOVE_ID (PHYSICAL), /** */ - BTREE_FORWARD_PAGE_SPLIT, + BTREE_FORWARD_PAGE_SPLIT (PHYSICAL), /** */ - BTREE_EXISTING_PAGE_SPLIT, + BTREE_EXISTING_PAGE_SPLIT (PHYSICAL), /** */ - BTREE_PAGE_MERGE, + BTREE_PAGE_MERGE (PHYSICAL), /** */ - PAGES_LIST_SET_NEXT, + PAGES_LIST_SET_NEXT (PHYSICAL), /** */ - PAGES_LIST_SET_PREVIOUS, + PAGES_LIST_SET_PREVIOUS (PHYSICAL), /** */ - PAGES_LIST_INIT_NEW_PAGE, + PAGES_LIST_INIT_NEW_PAGE (PHYSICAL), /** */ - PAGES_LIST_ADD_PAGE, + PAGES_LIST_ADD_PAGE (PHYSICAL), /** */ - PAGES_LIST_REMOVE_PAGE, + PAGES_LIST_REMOVE_PAGE (PHYSICAL), /** */ - META_PAGE_INIT, + META_PAGE_INIT (PHYSICAL), /** */ - PARTITION_META_PAGE_UPDATE_COUNTERS, + PARTITION_META_PAGE_UPDATE_COUNTERS (PHYSICAL), /** Memory recovering start marker */ MEMORY_RECOVERY, /** */ - TRACKING_PAGE_DELTA, + TRACKING_PAGE_DELTA (PHYSICAL), /** Meta page update last successful snapshot id. */ - META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID, + META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID (MIXED), /** Meta page update last successful full snapshot id. */ - META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID, + META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID (MIXED), /** Meta page update next snapshot id. */ - META_PAGE_UPDATE_NEXT_SNAPSHOT_ID, + META_PAGE_UPDATE_NEXT_SNAPSHOT_ID (MIXED), /** Meta page update last allocated index. */ - META_PAGE_UPDATE_LAST_ALLOCATED_INDEX, + META_PAGE_UPDATE_LAST_ALLOCATED_INDEX (MIXED), /** Partition meta update state. */ - PART_META_UPDATE_STATE, + PART_META_UPDATE_STATE (MIXED), /** Page list meta reset count record. */ - PAGE_LIST_META_RESET_COUNT_RECORD, + PAGE_LIST_META_RESET_COUNT_RECORD (PHYSICAL), /** Switch segment record. * Marker record for indicate end of segment. @@ -160,22 +163,22 @@ public enum RecordType { * that one byte in the end,then we write SWITCH_SEGMENT_RECORD as marker end of segment. * No need write CRC or WAL pointer for this record. It is byte marker record. * */ - SWITCH_SEGMENT_RECORD, + SWITCH_SEGMENT_RECORD (INTERNAL), /** */ - DATA_PAGE_UPDATE_RECORD, + DATA_PAGE_UPDATE_RECORD (PHYSICAL), /** init */ - BTREE_META_PAGE_INIT_ROOT2, + BTREE_META_PAGE_INIT_ROOT2 (PHYSICAL), /** Partition destroy. */ - PARTITION_DESTROY, + PARTITION_DESTROY (PHYSICAL), /** Snapshot record. */ SNAPSHOT, /** Metastore data record. */ - METASTORE_DATA_RECORD, + METASTORE_DATA_RECORD (LOGICAL), /** Exchange record. */ EXCHANGE, @@ -184,28 +187,57 @@ public enum RecordType { RESERVED, /** Rotated id part record. */ - ROTATED_ID_PART_RECORD, + ROTATED_ID_PART_RECORD (PHYSICAL), /** */ - MVCC_DATA_PAGE_MARK_UPDATED_RECORD, + MVCC_DATA_PAGE_MARK_UPDATED_RECORD (PHYSICAL), /** */ - MVCC_DATA_PAGE_TX_STATE_HINT_UPDATED_RECORD, + MVCC_DATA_PAGE_TX_STATE_HINT_UPDATED_RECORD (PHYSICAL), /** */ - MVCC_DATA_PAGE_NEW_TX_STATE_HINT_UPDATED_RECORD, + MVCC_DATA_PAGE_NEW_TX_STATE_HINT_UPDATED_RECORD (PHYSICAL), /** Encrypted WAL-record. */ - ENCRYPTED_RECORD, + ENCRYPTED_RECORD (PHYSICAL), /** Ecnrypted data record. */ - ENCRYPTED_DATA_RECORD, + ENCRYPTED_DATA_RECORD (LOGICAL), /** Mvcc data record. */ - MVCC_DATA_RECORD, + MVCC_DATA_RECORD (LOGICAL), /** Mvcc Tx state change record. */ - MVCC_TX_RECORD; + MVCC_TX_RECORD (LOGICAL); + + /** + * When you're adding a new record don't forget to choose record purpose explicitly + * if record is needed for physical or logical recovery. + * By default the purpose of record is {@link RecordPurpose#CUSTOM} and this record will not be used in recovery process. + * For more information read description of {@link RecordPurpose}. + */ + private final RecordPurpose purpose; + + /** + * @param purpose Purpose. + */ + RecordType(RecordPurpose purpose) { + this.purpose = purpose; + } + + /** + * Default constructor. + */ + RecordType() { + this(CUSTOM); + } + + /** + * @return Purpose of record. + */ + public RecordPurpose purpose() { + return purpose; + } /** */ private static final RecordType[] VALS = RecordType.values(); @@ -223,6 +255,37 @@ public static RecordType fromOrdinal(int ord) { public static final int STOP_ITERATION_RECORD_TYPE = 0; } + /** + * Record purposes set. + */ + public enum RecordPurpose { + /** + * Internal records are needed for correct iterating over WAL structure. + * These records will never be returned to user during WAL iteration. + */ + INTERNAL, + /** + * Physical records are needed for correct recovering physical state of {@link org.apache.ignite.internal.pagemem.PageMemory}. + * {@link org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager#restoreBinaryMemory(org.apache.ignite.lang.IgnitePredicate, org.apache.ignite.lang.IgniteBiPredicate)}. + */ + PHYSICAL, + /** + * Logical records are needed to replay logical updates since last checkpoint. + * {@link GridCacheDatabaseSharedManager#applyLogicalUpdates(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.CheckpointStatus, org.apache.ignite.lang.IgnitePredicate, org.apache.ignite.lang.IgniteBiPredicate, boolean)} + */ + LOGICAL, + /** + * Physical-logical records are used both for physical and logical recovery. + * Usually these records contain meta-information about partitions. + * NOTE: Not recommend to use this type without strong reason. + */ + MIXED, + /** + * Custom records are needed for any custom iterations over WAL in various components. + */ + CUSTOM + } + /** */ private int size; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 6d324b2747568..1d89b83ccf4a8 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -40,6 +40,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -173,6 +174,7 @@ import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD; +import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD; import static org.apache.ignite.internal.util.IgniteUtils.checkpointBufferSize; /** @@ -718,11 +720,11 @@ private void readMetastore() throws IgniteCheckedException { try { dataRegion(METASTORE_DATA_REGION_NAME).pageMemory().start(); - performBinaryMemoryRestore(status, g -> MetaStorage.METASTORAGE_CACHE_ID == g, false); + performBinaryMemoryRestore(status, onlyMetastorageGroup(), physicalRecords(), false); metaStorage = createMetastorage(true); - applyLogicalUpdates(status, g -> MetaStorage.METASTORAGE_CACHE_ID == g, false, true); + applyLogicalUpdates(status, onlyMetastorageGroup(), onlyMetastorageRecords(), false); fillWalDisabledGroups(); @@ -905,10 +907,14 @@ private MetaStorage createMetastorage(boolean readOnly) throws IgniteCheckedExce /** * @param cacheGroupsPredicate Cache groups to restore. + * @param recordTypePredicate Filter records by type. * @return Last seen WAL pointer during binary memory recovery. * @throws IgniteCheckedException If failed. */ - private RestoreBinaryState restoreBinaryMemory(Predicate cacheGroupsPredicate) throws IgniteCheckedException { + private RestoreBinaryState restoreBinaryMemory( + IgnitePredicate cacheGroupsPredicate, + IgniteBiPredicate recordTypePredicate + ) throws IgniteCheckedException { long time = System.currentTimeMillis(); try { @@ -921,9 +927,14 @@ private RestoreBinaryState restoreBinaryMemory(Predicate cacheGroupsPre // First, bring memory to the last consistent checkpoint state if needed. // This method should return a pointer to the last valid record in the WAL. - RestoreBinaryState binaryState = performBinaryMemoryRestore(status, cacheGroupsPredicate, true); + RestoreBinaryState binaryState = performBinaryMemoryRestore( + status, + cacheGroupsPredicate, + recordTypePredicate, + true + ); - WALPointer restored = binaryState.lastReadRecordPointer(); + WALPointer restored = binaryState.lastReadRecordPointer().map(FileWALPointer::next).orElse(null); if (restored == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { throw new StorageException("The memory cannot be restored. The critical part of WAL archive is missing " + @@ -1905,7 +1916,8 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC // Restore binary memory for all not WAL disabled cache groups. restoreBinaryMemory( - g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g) + groupsWithEnabledWal(), + physicalRecords() ); if (recoveryVerboseLogging && log.isInfoEnabled()) { @@ -1917,10 +1929,10 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC CheckpointStatus status = readCheckpointStatus(); RestoreLogicalState logicalState = applyLogicalUpdates( - status, - g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g), - true, - false + status, + groupsWithEnabledWal(), + logicalRecords(), + true ); if (recoveryVerboseLogging && log.isInfoEnabled()) { @@ -1929,7 +1941,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC dumpPartitionsInfo(cctx, log); } - walTail = tailPointer(logicalState.lastRead); + walTail = tailPointer(logicalState.lastReadRecordPointer().orElse(null)); cctx.wal().onDeActivate(kctx); } @@ -1951,20 +1963,21 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC * @throws IgniteCheckedException If failed. */ private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { - WALPointer lastRead = from; + WALIterator it = cctx.wal().replay(from); - try (WALIterator it = cctx.wal().replay(from)) { + try { while (it.hasNextX()) { IgniteBiTuple rec = it.nextX(); if (rec == null) break; - - lastRead = rec.get1(); } } + finally { + it.close(); + } - return lastRead != null ? lastRead.next() : null; + return it.lastRead().map(WALPointer::next).orElse(null); } /** @@ -2000,7 +2013,8 @@ private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { */ private RestoreBinaryState performBinaryMemoryRestore( CheckpointStatus status, - Predicate cacheGroupsPredicate, + IgnitePredicate cacheGroupsPredicate, + IgniteBiPredicate recordTypePredicate, boolean finalizeState ) throws IgniteCheckedException { if (log.isInfoEnabled()) @@ -2023,13 +2037,15 @@ private RestoreBinaryState performBinaryMemoryRestore( long lastArchivedSegment = cctx.wal().lastArchivedSegment(); - RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, cacheGroupsPredicate); + WALIterator it = cctx.wal().replay(status.endPtr, recordTypePredicate); + + RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, it, lastArchivedSegment, cacheGroupsPredicate); int applied = 0; - try (WALIterator it = cctx.wal().replay(status.endPtr)) { + try { while (it.hasNextX()) { - WALRecord rec = restoreBinaryState.next(it); + WALRecord rec = restoreBinaryState.next(); if (rec == null) break; @@ -2143,11 +2159,14 @@ private RestoreBinaryState performBinaryMemoryRestore( } } } + finally { + it.close(); + } if (!finalizeState) return null; - FileWALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); + FileWALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer().orElse(null); if (status.needRestoreMemory()) { if (restoreBinaryState.needApplyBinaryUpdate()) @@ -2164,9 +2183,6 @@ private RestoreBinaryState performBinaryMemoryRestore( cpHistory.initialize(retreiveHistory()); - // Move pointer position to the end of last read record. - restoreBinaryState.lastRead = lastReadPtr != null ? lastReadPtr.next() : lastReadPtr; - return restoreBinaryState; } @@ -2224,7 +2240,7 @@ public void applyUpdatesOnRecovery( switch (rec.type()) { case MVCC_DATA_RECORD: - case DATA_RECORD: + case DATA_RECORD: checkpointReadLock(); try { @@ -2260,24 +2276,24 @@ else if (log != null) break; - case MVCC_TX_RECORD: - checkpointReadLock(); + case MVCC_TX_RECORD: + checkpointReadLock(); - try { - MvccTxRecord txRecord = (MvccTxRecord)rec; + try { + MvccTxRecord txRecord = (MvccTxRecord)rec; - byte txState = convertToTxState(txRecord.state()); + byte txState = convertToTxState(txRecord.state()); - cctx.coordinators().updateState(txRecord.mvccVersion(), txState, false); - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } - finally { - checkpointReadUnlock(); - } + cctx.coordinators().updateState(txRecord.mvccVersion(), txState, false); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + finally { + checkpointReadUnlock(); + } - break; + break; default: // Skip other records. @@ -2293,9 +2309,9 @@ else if (log != null) */ private RestoreLogicalState applyLogicalUpdates( CheckpointStatus status, - Predicate cacheGroupsPredicate, - boolean skipFieldLookup, - boolean metaStoreOnly + IgnitePredicate cacheGroupsPredicate, + IgniteBiPredicate recordTypePredicate, + boolean skipFieldLookup ) throws IgniteCheckedException { if (log.isInfoEnabled()) log.info("Applying lost cache updates since last checkpoint record [lastMarked=" @@ -2304,17 +2320,19 @@ private RestoreLogicalState applyLogicalUpdates( if (skipFieldLookup) cctx.kernalContext().query().skipFieldLookup(true); - long lastArchivedSegment = cctx.wal().lastArchivedSegment(); - - RestoreLogicalState restoreLogicalState = new RestoreLogicalState(lastArchivedSegment, cacheGroupsPredicate); - long start = U.currentTimeMillis(); int applied = 0; - try (WALIterator it = cctx.wal().replay(status.startPtr)) { + long lastArchivedSegment = cctx.wal().lastArchivedSegment(); + + WALIterator it = cctx.wal().replay(status.startPtr, recordTypePredicate); + + RestoreLogicalState restoreLogicalState = new RestoreLogicalState(it, lastArchivedSegment, cacheGroupsPredicate); + + try { while (it.hasNextX()) { - WALRecord rec = restoreLogicalState.next(it); + WALRecord rec = restoreLogicalState.next(); if (rec == null) break; @@ -2322,6 +2340,7 @@ private RestoreLogicalState applyLogicalUpdates( switch (rec.type()) { case MVCC_DATA_RECORD: case DATA_RECORD: + case ENCRYPTED_DATA_RECORD: DataRecord dataRec = (DataRecord)rec; for (DataEntry dataEntry : dataRec.writeEntries()) { @@ -2342,6 +2361,15 @@ private RestoreLogicalState applyLogicalUpdates( break; + case MVCC_TX_RECORD: + MvccTxRecord txRecord = (MvccTxRecord)rec; + + byte txState = convertToTxState(txRecord.state()); + + cctx.coordinators().updateState(txRecord.mvccVersion(), txState, false); + + break; + case PART_META_UPDATE_STATE: PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec; @@ -2367,6 +2395,7 @@ private RestoreLogicalState applyLogicalUpdates( case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID: case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID: + case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX: PageDeltaRecord rec0 = (PageDeltaRecord) rec; PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId()); @@ -2392,24 +2421,14 @@ private RestoreLogicalState applyLogicalUpdates( break; - case MVCC_TX_RECORD: - if (metaStoreOnly) - continue; - - MvccTxRecord txRecord = (MvccTxRecord)rec; - - byte txState = convertToTxState(txRecord.state()); - - cctx.coordinators().updateState(txRecord.mvccVersion(), txState, false); - - break; - default: // Skip other records. } } } finally { + it.close(); + if (skipFieldLookup) cctx.kernalContext().query().skipFieldLookup(false); } @@ -4609,6 +4628,18 @@ private static void dumpPartitionsInfo(CacheGroupContext grp, IgniteLogger log) assert pageStore != null : "Persistent cache should have initialize page store manager."; for (int p = 0; p < grp.affinity().partitions(); p++) { + if (grp.topology().localPartition(p) != null) { + GridDhtLocalPartition part = grp.topology().localPartition(p); + + log.info("Partition [grp=" + grp.cacheOrGroupName() + + ", id=" + p + + ", state=" + part.state() + + ", counter=" + part.updateCounter() + + ", size=" + part.fullSize() + "]"); + + continue; + } + if (!pageStore.exists(grp.groupId(), p)) continue; @@ -4656,13 +4687,16 @@ private static void dumpPartitionsInfo(CacheGroupContext grp, IgniteLogger log) * Recovery lifecycle for read-write metastorage. */ private class MetastorageRecoveryLifecycle implements DatabaseLifecycleListener { + /** {@inheritDoc} */ @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException { cctx.pageStore().initializeForMetastorage(); } + /** {@inheritDoc} */ @Override public void afterBinaryMemoryRestore( IgniteCacheDatabaseSharedManager mgr, - RestoreBinaryState restoreState) throws IgniteCheckedException { + RestoreBinaryState restoreState + ) throws IgniteCheckedException { assert metaStorage == null; metaStorage = createMetastorage(false); @@ -4670,44 +4704,84 @@ private class MetastorageRecoveryLifecycle implements DatabaseLifecycleListener } /** - * Abstract class for create restore context. + * @return Cache group predicate that passes only Metastorage cache group id. + */ + private IgnitePredicate onlyMetastorageGroup() { + return groupId -> MetaStorage.METASTORAGE_CACHE_ID == groupId; + } + + /** + * @return Cache group predicate that passes only cache groups with enabled WAL. + */ + private IgnitePredicate groupsWithEnabledWal() { + return groupId -> !initiallyGlobalWalDisabledGrps.contains(groupId) + && !initiallyLocalWalDisabledGrps.contains(groupId); + } + + /** + * @return WAL records predicate that passes only Metastorage data records. + */ + private IgniteBiPredicate onlyMetastorageRecords() { + return (type, ptr) -> type == METASTORE_DATA_RECORD; + } + + /** + * @return WAL records predicate that passes only physical and mixed WAL records. + */ + private IgniteBiPredicate physicalRecords() { + return (type, ptr) -> type.purpose() == WALRecord.RecordPurpose.PHYSICAL + || type.purpose() == WALRecord.RecordPurpose.MIXED; + } + + /** + * @return WAL records predicate that passes only logical and mixed WAL records. + */ + private IgniteBiPredicate logicalRecords() { + return (type, ptr) -> type.purpose() == WALRecord.RecordPurpose.LOGICAL + || type.purpose() == WALRecord.RecordPurpose.MIXED; + } + + /** + * Abstract class to create restore context. */ private abstract class RestoreStateContext { /** Last archived segment. */ protected final long lastArchivedSegment; - /** Last read record WAL pointer. */ - protected FileWALPointer lastRead; + /** WAL iterator. */ + private final WALIterator iterator; /** Only {@link WalRecordCacheGroupAware} records satisfied this predicate will be applied. */ - private final Predicate cacheGroupPredicate; - - /** Set to {@code true} if data records should be skipped. */ - private final boolean skipDataRecords; + private final IgnitePredicate cacheGroupPredicate; /** + * @param iterator WAL iterator. * @param lastArchivedSegment Last archived segment index. + * @param cacheGroupPredicate Cache groups predicate. */ - public RestoreStateContext(long lastArchivedSegment, Predicate cacheGroupPredicate, boolean skipDataRecords) { + protected RestoreStateContext( + WALIterator iterator, + long lastArchivedSegment, + IgnitePredicate cacheGroupPredicate + ) { + this.iterator = iterator; this.lastArchivedSegment = lastArchivedSegment; this.cacheGroupPredicate = cacheGroupPredicate; - this.skipDataRecords = skipDataRecords; } /** * Advance iterator to the next record. * - * @param it WAL iterator. * @return WALRecord entry. * @throws IgniteCheckedException If CRC check fail during binary recovery state or another exception occurring. */ - public WALRecord next(WALIterator it) throws IgniteCheckedException { + public WALRecord next() throws IgniteCheckedException { try { for (;;) { - if (!it.hasNextX()) + if (!iterator.hasNextX()) return null; - IgniteBiTuple tup = it.nextX(); + IgniteBiTuple tup = iterator.nextX(); if (tup == null) return null; @@ -4716,45 +4790,19 @@ public WALRecord next(WALIterator it) throws IgniteCheckedException { WALPointer ptr = tup.get1(); - lastRead = (FileWALPointer)ptr; - rec.position(ptr); - // Filter out records. + // Filter out records by group id. if (rec instanceof WalRecordCacheGroupAware) { - WalRecordCacheGroupAware groupAwareRecord = (WalRecordCacheGroupAware) rec; + WalRecordCacheGroupAware grpAwareRecord = (WalRecordCacheGroupAware) rec; - if (!cacheGroupPredicate.test(groupAwareRecord.groupId())) + if (!cacheGroupPredicate.apply(grpAwareRecord.groupId())) continue; } - switch (rec.type()) { - case METASTORE_DATA_RECORD: - case MVCC_DATA_RECORD: - case DATA_RECORD: - if (skipDataRecords) - continue; - - if (rec instanceof DataRecord) { - DataRecord dataRecord = (DataRecord) rec; - - // Filter data entries by group id. - List filteredEntries = dataRecord.writeEntries().stream() - .filter(entry -> { - int cacheId = entry.cacheId(); - - return cctx.cacheContext(cacheId) != null && cacheGroupPredicate.test(cctx.cacheContext(cacheId).groupId()); - }) - .collect(Collectors.toList()); - - dataRecord.setWriteEntries(filteredEntries); - } - - break; - - default: - break; - } + // Filter out data entries by group id. + if (rec instanceof DataRecord) + rec = filterEntriesByGroupId((DataRecord) rec); return rec; } @@ -4775,22 +4823,38 @@ public WALRecord next(WALIterator it) throws IgniteCheckedException { } } + /** + * Filter outs data entries from given data record that not satisfy {@link #cacheGroupPredicate}. + * + * @param record Original data record. + * @return Data record with filtered data entries. + */ + private DataRecord filterEntriesByGroupId(DataRecord record) { + List filteredEntries = record.writeEntries().stream() + .filter(entry -> { + int cacheId = entry.cacheId(); + + return cctx.cacheContext(cacheId) != null && cacheGroupPredicate.apply(cctx.cacheContext(cacheId).groupId()); + }) + .collect(Collectors.toList()); + + return record.setWriteEntries(filteredEntries); + } + /** * * @return Last read WAL record pointer. */ - public FileWALPointer lastReadRecordPointer() { - return lastRead; + public Optional lastReadRecordPointer() { + return iterator.lastRead().map(ptr -> (FileWALPointer)ptr); } /** * * @return Flag indicates need throws CRC exception or not. */ - public boolean throwsCRCError(){ - FileWALPointer lastReadPtr = lastRead; - - return lastReadPtr != null && lastReadPtr.index() <= lastArchivedSegment; + public boolean throwsCRCError() { + return lastReadRecordPointer().filter(ptr -> ptr.index() <= lastArchivedSegment).isPresent(); } } @@ -4806,10 +4870,17 @@ public class RestoreBinaryState extends RestoreStateContext { /** * @param status Checkpoint status. + * @param iterator WAL iterator. * @param lastArchivedSegment Last archived segment index. + * @param cacheGroupsPredicate Cache groups predicate. */ - public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, Predicate cacheGroupsPredicate) { - super(lastArchivedSegment, cacheGroupsPredicate, true); + public RestoreBinaryState( + CheckpointStatus status, + WALIterator iterator, + long lastArchivedSegment, + IgnitePredicate cacheGroupsPredicate + ) { + super(iterator, lastArchivedSegment, cacheGroupsPredicate); this.status = status; this.needApplyBinaryUpdates = status.needRestoreMemory(); @@ -4818,12 +4889,11 @@ public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, Pre /** * Advance iterator to the next record. * - * @param it WAL iterator. * @return WALRecord entry. * @throws IgniteCheckedException If CRC check fail during binary recovery state or another exception occurring. */ - @Override public WALRecord next(WALIterator it) throws IgniteCheckedException { - WALRecord rec = super.next(it); + @Override public WALRecord next() throws IgniteCheckedException { + WALRecord rec = super.next(); if (rec == null) return null; @@ -4860,7 +4930,7 @@ public boolean needApplyBinaryUpdate() { */ @Override public boolean throwsCRCError() { log.info("Throws CRC error check [needApplyBinaryUpdates=" + needApplyBinaryUpdates + - ", lastArchivedSegment=" + lastArchivedSegment + ", lastRead=" + lastRead + ']'); + ", lastArchivedSegment=" + lastArchivedSegment + ", lastRead=" + lastReadRecordPointer() + ']'); if (needApplyBinaryUpdates) return true; @@ -4879,8 +4949,8 @@ public class RestoreLogicalState extends RestoreStateContext { /** * @param lastArchivedSegment Last archived segment index. */ - public RestoreLogicalState(long lastArchivedSegment, Predicate cacheGroupsPredicate) { - super(lastArchivedSegment, cacheGroupsPredicate, false); + public RestoreLogicalState(WALIterator iterator, long lastArchivedSegment, IgnitePredicate cacheGroupsPredicate) { + super(iterator, lastArchivedSegment, cacheGroupsPredicate); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 04255c01e08e2..f78428d055e52 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -194,6 +194,17 @@ public IndexStorage getIndexStorage() { } } + syncMetadata(execSvc, ctx, needSnapshot); + } + + /** + * Syncs and saves meta-information of all data structures to page memory. + * + * @param execSvc Executor service to run save process + * @param ctx Checkpoint listener context. + * @throws IgniteCheckedException If failed. + */ + private void syncMetadata(Executor execSvc, Context ctx, boolean needSnapshot) throws IgniteCheckedException { if (execSvc == null) { reuseList.saveMetadata(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 3cbe577c24c59..f37b154c60507 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -22,6 +22,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteOrder; +import java.util.Optional; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.pagemem.wal.WALIterator; @@ -92,6 +93,9 @@ public abstract class AbstractWalRecordsIterator /** Factory to provide I/O interfaces for read primitives with files. */ private final SegmentFileInputFactory segmentFileInputFactory; + /** Position of last read valid record. */ + private WALPointer lastRead; + /** * @param log Logger. * @param sharedCtx Shared context. @@ -154,6 +158,8 @@ protected void advance() throws IgniteCheckedException { curRec = advanceRecord(currWalSegment); if (curRec != null) { + lastRead = curRec.get1(); + if (curRec.get2().type() == null) continue; // Record was skipped by filter of current serializer, should read next record. @@ -183,6 +189,11 @@ protected void advance() throws IgniteCheckedException { } } + /** {@inheritDoc} */ + @Override public Optional lastRead() { + return Optional.ofNullable(lastRead); + } + /** * @param tailReachedException Tail reached exception. * @param currWalSegment Current WAL segment read handler. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index b428721305716..819317ea08264 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -124,6 +124,7 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -934,6 +935,14 @@ private FileWriteHandle closeBufAndRollover( /** {@inheritDoc} */ @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException { + return replay(start, null); + } + + /** {@inheritDoc} */ + @Override public WALIterator replay( + WALPointer start, + @Nullable IgniteBiPredicate recordDeserializeFilter + ) throws IgniteCheckedException, StorageException { assert start == null || start instanceof FileWALPointer : "Invalid start pointer: " + start; FileWriteHandle hnd = currentHandle(); @@ -950,7 +959,7 @@ private FileWriteHandle closeBufAndRollover( (FileWALPointer)start, end, dsCfg, - new RecordSerializerFactoryImpl(cctx), + new RecordSerializerFactoryImpl(cctx).recordDeserializeFilter(recordDeserializeFilter), ioFactory, archiver, decompressor, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index b1ad8ddca5a4b..c961f5201e2a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -116,6 +116,7 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; +import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -800,8 +801,15 @@ else if (rolloverType == RolloverType.CURRENT_SEGMENT) { } /** {@inheritDoc} */ - @Override public WALIterator replay(WALPointer start) - throws IgniteCheckedException, StorageException { + @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException { + return replay(start, null); + } + + /** {@inheritDoc} */ + @Override public WALIterator replay( + WALPointer start, + @Nullable IgniteBiPredicate recordDeserializeFilter + ) throws IgniteCheckedException, StorageException { assert start == null || start instanceof FileWALPointer : "Invalid start pointer: " + start; FileWriteHandle hnd = currentHandle(); @@ -818,7 +826,7 @@ else if (rolloverType == RolloverType.CURRENT_SEGMENT) { (FileWALPointer)start, end, dsCfg, - new RecordSerializerFactoryImpl(cctx), + new RecordSerializerFactoryImpl(cctx).recordDeserializeFilter(recordDeserializeFilter), ioFactory, archiver, decompressor, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java index c149817299f25..96b78e664cc83 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.lang.IgniteBiPredicate; +import org.jetbrains.annotations.Nullable; /** * @@ -34,7 +35,7 @@ public class RecordSerializerFactoryImpl implements RecordSerializerFactory { private boolean needWritePointer; /** Read record filter. */ - private IgniteBiPredicate recordDeserializeFilter; + private @Nullable IgniteBiPredicate recordDeserializeFilter; /** * Marshalled mode flag. @@ -56,7 +57,7 @@ public RecordSerializerFactoryImpl(GridCacheSharedContext cctx) { */ public RecordSerializerFactoryImpl( GridCacheSharedContext cctx, - IgniteBiPredicate readTypeFilter + @Nullable IgniteBiPredicate readTypeFilter ) { this.cctx = cctx; this.recordDeserializeFilter = readTypeFilter; @@ -114,7 +115,7 @@ public IgniteBiPredicate recordDeserializeFilt /** {@inheritDoc} */ @Override public RecordSerializerFactoryImpl recordDeserializeFilter( - IgniteBiPredicate readTypeFilter + @Nullable IgniteBiPredicate readTypeFilter ) { this.recordDeserializeFilter = readTypeFilter; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index 31efa1f0f1e23..fa32c87709ee7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -144,7 +144,8 @@ public class RecordV1Serializer implements RecordSerializer { rec.position(ptr); - if (recordFilter != null && !recordFilter.apply(rec.type(), ptr)) + if (recType.purpose() != WALRecord.RecordPurpose.INTERNAL + && recordFilter != null && !recordFilter.apply(rec.type(), ptr)) return FilteredRecord.INSTANCE; else if (marshalledMode) { ByteBuffer buf = heapTlb.get(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java index d27a331141e32..0d78d08291bc7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -122,7 +122,8 @@ public class RecordV2Serializer implements RecordSerializer { ", expected pointer [idx=" + exp.index() + ", offset=" + exp.fileOffset() + "]"); } - if (recordFilter != null && !recordFilter.apply(recType, ptr)) { + if (recType.purpose() != WALRecord.RecordPurpose.INTERNAL + && recordFilter != null && !recordFilter.apply(recType, ptr)) { int toSkip = ptr.length() - REC_TYPE_SIZE - FILE_WAL_POINTER_SIZE - CRC_SIZE; assert toSkip >= 0 : "Too small saved record length: " + ptr; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java index 93cc074e6d6ab..530e4c4fb975d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Predicate; @@ -49,6 +50,8 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; @@ -253,7 +256,7 @@ public void testRecoveryOnDynamicallyStartedCaches() throws Exception { * */ public void testRecoveryWithMvccCaches() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10052"); + fail("https://issues.apache.org/jira/browse/IGNITE-10582"); List dynamicCaches = Lists.newArrayList( cacheConfiguration(DYNAMIC_CACHE_PREFIX + 0, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT), @@ -399,13 +402,21 @@ private void checkNoRebalanceAfterRecovery() { List nodes = G.allGrids(); - for (Ignite node : nodes) { + for (final Ignite node : nodes) { TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(node); + Set mvccCaches = ((IgniteEx) node).context().cache().cacheGroups().stream() + .flatMap(group -> group.caches().stream()) + .filter(cache -> cache.config().getAtomicityMode() == CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) + .map(GridCacheContext::groupId) + .collect(Collectors.toSet()); + List rebalancedGroups = spi.recordedMessages(true).stream() .map(msg -> (GridDhtPartitionDemandMessage) msg) - .map(msg -> msg.groupId()) + .map(GridCacheGroupIdMessage::groupId) .filter(grpId -> grpId != sysCacheGroupId) + //TODO: remove following filter when failover for MVCC will be fixed. + .filter(grpId -> !mvccCaches.contains(grpId)) .distinct() .collect(Collectors.toList()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java index ea3ed2f8ae0f9..8d854bb32117a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java @@ -26,7 +26,9 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.StorageException; +import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteFuture; +import org.jetbrains.annotations.Nullable; /** * @@ -72,6 +74,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { return null; } + /** {@inheritDoc} */ + @Override public WALIterator replay(WALPointer start, @Nullable IgniteBiPredicate recordDeserializeFilter) throws IgniteCheckedException, StorageException { + return null; + } + /** {@inheritDoc} */ @Override public boolean reserve(WALPointer start) { return false; From 68a6d9a83620b923be431c2cbcbf5342c559a03c Mon Sep 17 00:00:00 2001 From: Alexander Kalinin Date: Wed, 12 Dec 2018 22:26:09 +0700 Subject: [PATCH 231/403] IGNITE-10661 Fixed Docker image for E2E tests. (cherry picked from commit e8d0831ffbcb80ae2dda9b5c1e187938c7668717) --- modules/web-console/frontend/test/ci/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/web-console/frontend/test/ci/Dockerfile b/modules/web-console/frontend/test/ci/Dockerfile index 2573534dfb1d3..7b1d602765446 100644 --- a/modules/web-console/frontend/test/ci/Dockerfile +++ b/modules/web-console/frontend/test/ci/Dockerfile @@ -15,10 +15,12 @@ # limitations under the License. # -FROM alpine:edge +FROM node:8-alpine + +ENV NPM_CONFIG_LOGLEVEL error RUN apk --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ add \ - nodejs nodejs-npm chromium xwininfo xvfb dbus eudev ttf-freefont fluxbox + chromium xwininfo xvfb dbus eudev ttf-freefont fluxbox ENV CHROME_BIN /usr/bin/chromium-browser From baaae9d1488f398f243c2ac78c52e7ad0804768d Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Mon, 10 Dec 2018 14:23:02 +0300 Subject: [PATCH 232/403] IGNITE-10167: MVCC: MVCC-compatible IgniteCache.localEntries. This closes #5357. (cherry picked from commit 90207e987889d0b68c9f3d54f68bf6148fb37221) --- .../cache/CacheEvictableEntryImpl.java | 4 +- .../processors/cache/GridCacheAdapter.java | 65 ++++--------------- .../cache/GridCacheDeploymentManager.java | 2 +- .../processors/cache/GridCacheEntryEx.java | 3 +- .../processors/cache/GridCacheMapEntry.java | 6 +- .../processors/cache/GridCacheProxyImpl.java | 5 +- .../cache/IgniteCacheOffheapManager.java | 14 ++-- .../cache/IgniteCacheOffheapManagerImpl.java | 19 ++---- .../cache/IgniteCacheProxyImpl.java | 2 +- .../processors/cache/IgniteInternalCache.java | 3 +- .../near/GridNearCacheAdapter.java | 2 +- .../distributed/near/GridNearTxRemote.java | 4 +- .../processors/task/GridTaskProcessor.java | 29 ++++----- .../GridCachePreloadingEvictionsSelfTest.java | 2 +- .../cache/GridCacheTestEntryEx.java | 3 +- .../cache/IgniteCacheTxPeekModesTest.java | 8 --- .../GridCacheDhtPreloadDisabledSelfTest.java | 4 +- .../dht/GridCacheTxNodeFailureSelfTest.java | 4 +- .../near/GridCacheNearMultiNodeSelfTest.java | 6 +- .../igfs/IgfsProcessorSelfTest.java | 27 ++++---- .../junits/common/GridCommonAbstractTest.java | 4 +- .../testsuites/IgniteCacheMvccTestSuite.java | 38 ----------- .../cache/hibernate/HibernateCacheProxy.java | 6 +- 23 files changed, 83 insertions(+), 177 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java index bcaf890ab2f89..b7c882372ed03 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java @@ -87,7 +87,7 @@ protected CacheEvictableEntryImpl(GridCacheEntryEx cached) { */ @Nullable public V peek() { try { - CacheObject val = cached.peek(null); + CacheObject val = cached.peek(); return val != null ? val.value(cached.context().cacheObjectContext(), false) : null; } @@ -151,7 +151,7 @@ protected CacheEvictableEntryImpl(GridCacheEntryEx cached) { return null; try { - CacheObject val = e.peek(null); + CacheObject val = e.peek(); return val != null ? val.value(cached.context().cacheObjectContext(), false) : null; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index b16f389a5f125..89081e798b1f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -784,10 +784,16 @@ public void onKernalStop() { IgniteCacheOffheapManager offheapMgr = ctx.isNear() ? ctx.near().dht().context().offheap() : ctx.offheap(); - its.add(offheapMgr.cacheEntriesIterator(ctx, modes.primary, modes.backup, topVer, ctx.keepBinary())); + MvccSnapshot mvccSnapshot = ctx.mvccEnabled() ? MvccUtils.MVCC_MAX_SNAPSHOT : null; + + its.add(offheapMgr + .cacheEntriesIterator(ctx, modes.primary, modes.backup, topVer, ctx.keepBinary(), mvccSnapshot)); } } else if (modes.heap) { + if (ctx.mvccEnabled()) + return F.emptyIterator(); + if (modes.near && ctx.isNear()) its.add(ctx.near().nearEntries().iterator()); @@ -812,10 +818,8 @@ else if (modes.heap) { } /** {@inheritDoc} */ - @SuppressWarnings("ForLoopReplaceableByForEach") - @Nullable @Override public final V localPeek(K key, - CachePeekMode[] peekModes, - @Nullable IgniteCacheExpiryPolicy plc) + @Override public final V localPeek(K key, + CachePeekMode[] peekModes) throws IgniteCheckedException { A.notNull(key, "key"); @@ -882,10 +886,8 @@ else if (modes.heap) { GridCacheContext ctx0; while (true) { - if (nearKey) { - ctx0 = context(); + if (nearKey) e = peekEx(key); - } else { ctx0 = ctx.isNear() ? ctx.near().dht().context() : ctx; e = modes.offheap ? ctx0.cache().entryEx(key) : ctx0.cache().peekEx(key); @@ -897,7 +899,7 @@ else if (modes.heap) { try { cacheVal = ctx.mvccEnabled() ? e.mvccPeek(modes.heap && !modes.offheap) - : e.peek(modes.heap, modes.offheap, topVer, plc); + : e.peek(modes.heap, modes.offheap, topVer, null); } catch (GridCacheEntryRemovedException ignore) { if (log.isDebugEnabled()) @@ -918,7 +920,7 @@ else if (modes.heap) { else { while (true) { try { - cacheVal = localCachePeek0(cacheKey, modes.heap, modes.offheap, plc); + cacheVal = localCachePeek0(cacheKey, modes.heap, modes.offheap); break; } @@ -940,7 +942,6 @@ else if (modes.heap) { * @param key Key. * @param heap Read heap flag. * @param offheap Read offheap flag. - * @param plc Optional expiry policy. * @return Value. * @throws GridCacheEntryRemovedException If entry removed. * @throws IgniteCheckedException If failed. @@ -948,8 +949,7 @@ else if (modes.heap) { @SuppressWarnings("ConstantConditions") @Nullable private CacheObject localCachePeek0(KeyCacheObject key, boolean heap, - boolean offheap, - IgniteCacheExpiryPolicy plc) + boolean offheap) throws GridCacheEntryRemovedException, IgniteCheckedException { assert ctx.isLocal(); assert heap || offheap; @@ -958,7 +958,7 @@ else if (modes.heap) { if (e != null) { try { - return e.peek(heap, offheap, AffinityTopologyVersion.NONE, plc); + return e.peek(heap, offheap, AffinityTopologyVersion.NONE, null); } finally { e.touch(null); @@ -4059,43 +4059,6 @@ IgniteInternalFuture globalLoadCacheAsync(@Nullable IgniteBiPredicate p return entrySet().iterator(); } - /** - * @param opCtx Cache operation context. - * @return JCache Iterator. - */ - private Iterator> localIteratorHonorExpirePolicy(final CacheOperationContext opCtx) { - return F.iterator(iterator(), - new IgniteClosure, Cache.Entry>() { - private IgniteCacheExpiryPolicy expiryPlc = - ctx.cache().expiryPolicy(opCtx != null ? opCtx.expiry() : null); - - @Override public Cache.Entry apply(Cache.Entry lazyEntry) { - CacheOperationContext prev = ctx.gate().enter(opCtx); - try { - V val = localPeek(lazyEntry.getKey(), CachePeekModes.ONHEAP_ONLY, expiryPlc); - - GridCacheVersion ver = null; - - try { - ver = lazyEntry.unwrap(GridCacheVersion.class); - } - catch (IllegalArgumentException e) { - log.error("Failed to unwrap entry version information", e); - } - - return new CacheEntryImpl<>(lazyEntry.getKey(), val, ver); - } - catch (IgniteCheckedException e) { - throw CU.convertToCacheException(e); - } - finally { - ctx.gate().leave(prev); - } - } - }, false - ); - } - /** {@inheritDoc} */ @Override public Iterator> scanIterator(boolean keepBinary, @Nullable IgniteBiPredicate p) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java index b34da627bd7d3..15af1f10d091d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java @@ -312,7 +312,7 @@ private boolean undeploy(ClassLoader ldr, GridCacheEntryEx e, GridCacheAdapter c Object val0; try { - CacheObject v = entry.peek(null); + CacheObject v = entry.peek(); key0 = key.value(cache.context().cacheObjectContext(), false); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index 319c134a85f88..8cef1763af868 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -742,12 +742,11 @@ public boolean tmLock(IgniteInternalTx tx, /** * Peeks into entry without loading value or updating statistics. * - * @param plc Expiry policy if TTL should be updated. * @return Value. * @throws GridCacheEntryRemovedException If entry has been removed. * @throws IgniteCheckedException If failed. */ - @Nullable public CacheObject peek(@Nullable IgniteCacheExpiryPolicy plc) + @Nullable public CacheObject peek() throws GridCacheEntryRemovedException, IgniteCheckedException; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 341fd5c30e881..a99cd90604bce 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3212,13 +3212,13 @@ int hash() { } /** {@inheritDoc} */ - @Nullable @Override public CacheObject peek(@Nullable IgniteCacheExpiryPolicy plc) + @Nullable @Override public CacheObject peek() throws GridCacheEntryRemovedException, IgniteCheckedException { IgniteInternalTx tx = cctx.tm().localTx(); AffinityTopologyVersion topVer = tx != null ? tx.topologyVersion() : cctx.affinity().affinityTopologyVersion(); - return peek(true, false, topVer, plc); + return peek(true, false, topVer, null); } /** @@ -4463,7 +4463,7 @@ protected void removeValue() throws IgniteCheckedException { return null; try { - return e.peek(null); + return e.peek(); } catch (GridCacheEntryRemovedException ignored) { // No-op. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java index 6b3ca568a4c72..fa9f3cb56f995 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java @@ -926,13 +926,12 @@ public IgniteInternalCache delegate() { /** {@inheritDoc} */ @Nullable @Override public V localPeek(K key, - CachePeekMode[] peekModes, - @Nullable IgniteCacheExpiryPolicy plc) + CachePeekMode[] peekModes) throws IgniteCheckedException { CacheOperationContext prev = gate.enter(opCtx); try { - return delegate.localPeek(key, peekModes, plc); + return delegate.localPeek(key, peekModes); } finally { gate.leave(prev); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index ed3697aac57b2..7f0fc3096d9a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -191,10 +191,11 @@ public void invoke(GridCacheContext cctx, KeyCacheObject key, GridDhtLocalPartit /** * @param cctx Cache context. * @param key Key. + * @param mvccSnapshot MVCC snapshot. * @return Cached row, if available, null otherwise. * @throws IgniteCheckedException If failed. */ - @Nullable public CacheDataRow mvccRead(GridCacheContext cctx, KeyCacheObject key, MvccSnapshot ver) + @Nullable public CacheDataRow mvccRead(GridCacheContext cctx, KeyCacheObject key, MvccSnapshot mvccSnapshot) throws IgniteCheckedException; /** @@ -460,11 +461,11 @@ public void remove( public int onUndeploy(ClassLoader ldr); /** - * * @param cacheId Cache ID. * @param primary Primary entries flag. * @param backup Backup entries flag. * @param topVer Topology version. + * @param mvccSnapshot MVCC snapshot. * @return Rows iterator. * @throws IgniteCheckedException If failed. */ @@ -523,20 +524,21 @@ public IgniteRebalanceIterator rebalanceIterator(IgniteDhtDemandedPartitionsMap /** * @param cctx Cache context. - * @param primary Primary entries flag. - * @param backup Backup entries flag. + * @param primary {@code True} if need to return primary entries. + * @param backup {@code True} if need to return backup entries. * @param topVer Topology version. * @param keepBinary Keep binary flag. + * @param mvccSnapshot MVCC snapshot. * @return Entries iterator. * @throws IgniteCheckedException If failed. */ - // TODO: MVCC> public GridCloseableIterator> cacheEntriesIterator( GridCacheContext cctx, final boolean primary, final boolean backup, final AffinityTopologyVersion topVer, - final boolean keepBinary) throws IgniteCheckedException; + final boolean keepBinary, + @Nullable final MvccSnapshot mvccSnapshot) throws IgniteCheckedException; /** * @param cacheId Cache ID. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 8b1578fe9fc04..2f632f1e4a874 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -687,13 +687,13 @@ private Iterator cacheData(boolean primary, boolean backup, Affi } /** {@inheritDoc} */ - @Nullable @Override public CacheDataRow mvccRead(GridCacheContext cctx, KeyCacheObject key, MvccSnapshot ver) + @Nullable @Override public CacheDataRow mvccRead(GridCacheContext cctx, KeyCacheObject key, MvccSnapshot mvccSnapshot) throws IgniteCheckedException { - assert ver != null; + assert mvccSnapshot != null; CacheDataStore dataStore = dataStore(cctx, key); - CacheDataRow row = dataStore != null ? dataStore.mvccFind(cctx, key, ver) : null; + CacheDataRow row = dataStore != null ? dataStore.mvccFind(cctx, key, mvccSnapshot) : null; assert row == null || row.value() != null : row; @@ -812,21 +812,16 @@ private Iterator cacheData(boolean primary, boolean backup, Affi return 0; } - /** - * @param primary {@code True} if need return primary entries. - * @param backup {@code True} if need return backup entries. - * @param topVer Topology version to use. - * @return Entries iterator. - * @throws IgniteCheckedException If failed. - */ + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public GridCloseableIterator> cacheEntriesIterator( final GridCacheContext cctx, final boolean primary, final boolean backup, final AffinityTopologyVersion topVer, - final boolean keepBinary) throws IgniteCheckedException { - final Iterator it = cacheIterator(cctx.cacheId(), primary, backup, topVer, null); + final boolean keepBinary, + @Nullable final MvccSnapshot mvccSnapshot) throws IgniteCheckedException { + final Iterator it = cacheIterator(cctx.cacheId(), primary, backup, topVer, mvccSnapshot); return new GridCloseableIteratorAdapter>() { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java index addd189d32c63..554e4a399e7a7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java @@ -816,7 +816,7 @@ private void validate(Query qry) { /** {@inheritDoc} */ @Nullable @Override public V localPeek(K key, CachePeekMode... peekModes) { try { - return delegate.localPeek(key, peekModes, null); + return delegate.localPeek(key, peekModes); } catch (IgniteException | IgniteCheckedException e) { throw cacheException(e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java index 9d40190434b3c..64e21e3fe5f32 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java @@ -302,11 +302,10 @@ public interface IgniteInternalCache extends Iterable> { /** * @param key Key. * @param peekModes Peek modes. - * @param plc Expiry policy if TTL should be updated. * @return Value. * @throws IgniteCheckedException If failed. */ - @Nullable public V localPeek(K key, CachePeekMode[] peekModes, @Nullable IgniteCacheExpiryPolicy plc) + @Nullable public V localPeek(K key, CachePeekMode[] peekModes) throws IgniteCheckedException; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java index 39047813e85e5..086247c519d15 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java @@ -444,7 +444,7 @@ private EntrySet(Set> nearSet, Set> dhtSet) F.iterator0(dhtSet, false, new P1>() { @Override public boolean apply(Cache.Entry e) { try { - return GridNearCacheAdapter.super.localPeek(e.getKey(), NEAR_PEEK_MODE, null) == null; + return GridNearCacheAdapter.super.localPeek(e.getKey(), NEAR_PEEK_MODE) == null; } catch (IgniteCheckedException ex) { throw new IgniteException(ex); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxRemote.java index 879bf26fb7273..cd0ee6c2f6e81 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxRemote.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxRemote.java @@ -340,7 +340,7 @@ private boolean addEntry(IgniteTxEntry entry) throws IgniteCheckedException { try { cached.unswap(); - CacheObject val = cached.peek(null); + CacheObject val = cached.peek(); if (val == null && cached.evictInternal(xidVer, null, false)) { evicted.add(entry.txKey()); @@ -401,7 +401,7 @@ public boolean addEntry( else { cached.unswap(); - CacheObject peek = cached.peek(null); + CacheObject peek = cached.peek(); if (peek == null && cached.evictInternal(xidVer, null, false)) { cached.context().cache().removeIfObsolete(key.key()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index f663d8fca8876..b81665e32002f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -17,6 +17,19 @@ package org.apache.ignite.internal.processors.task; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; @@ -68,20 +81,6 @@ import org.apache.ignite.plugin.security.SecurityPermission; import org.jetbrains.annotations.Nullable; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.LongAdder; - import static org.apache.ignite.events.EventType.EVT_MANAGEMENT_TASK_STARTED; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; @@ -497,7 +496,7 @@ public String resolveTaskName(int taskNameHash) { try { return taskMetaCache().localPeek( - new GridTaskNameHashKey(taskNameHash), null, null); + new GridTaskNameHashKey(taskNameHash), null); } catch (IgniteCheckedException e) { throw new IgniteException(e); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java index f9042ef39e3b2..2a10d9723641c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java @@ -234,7 +234,7 @@ private void checkCachesConsistency(Ignite ignite1, Ignite ignite2) throws Excep "Sizes do not match [s1=" + cache1.size(ALL_PEEK_MODES) + ", s2=" + cache2.size(ALL_PEEK_MODES) + ']'; for (Integer key : cache1.keySet()) { - Object e = cache1.localPeek(key, new CachePeekMode[] {CachePeekMode.ONHEAP}, null); + Object e = cache1.localPeek(key, new CachePeekMode[] {CachePeekMode.ONHEAP}); if (e != null) assert cache2.containsKey(key) : "Cache2 does not contain key: " + key; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java index 6fb200dcda924..358dfc391ae01 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java @@ -925,8 +925,7 @@ GridCacheMvccCandidate anyOwner() { } /** {@inheritDoc} */ - @Nullable @Override public CacheObject peek( - @Nullable IgniteCacheExpiryPolicy plc) + @Nullable @Override public CacheObject peek() throws GridCacheEntryRemovedException, IgniteCheckedException { return null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java index ea22f53b0b5bc..c6c937bd4698f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java @@ -41,12 +41,4 @@ public class IgniteCacheTxPeekModesTest extends IgniteCachePeekModesAbstractTest @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; } - - /** {@inheritDoc} */ - @Override public void testLocalPeek() throws Exception { - // TODO: uncomment and re-open ticket if fails. -// fail("https://issues.apache.org/jira/browse/IGNITE-1824"); - - super.testLocalPeek(); - } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java index d8bc44a850508..8899654e12cea 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java @@ -194,7 +194,7 @@ public void testDisabledPreloader() throws Exception { for (int i = 0; i < keyCnt; i++) { assertNull(near(cache1).peekEx(i)); - assertNotNull((dht(cache1).localPeek(i, null, null))); + assertNotNull((dht(cache1).localPeek(i, null))); assertEquals(Integer.toString(i), cache1.localPeek(i)); } @@ -288,4 +288,4 @@ private void putKeys(Cache c, int cnt) { for (int i = 0; i < cnt; i++) c.put(i, Integer.toString(i)); } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheTxNodeFailureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheTxNodeFailureSelfTest.java index a5c41a0b2deec..1c9f97df0cf2f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheTxNodeFailureSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheTxNodeFailureSelfTest.java @@ -345,14 +345,14 @@ private void dataCheck(IgniteKernal orig, IgniteKernal backup, int key, boolean assertNotNull(dhtEntry); assertTrue("dhtEntry=" + dhtEntry, dhtEntry.remoteMvccSnapshot().isEmpty()); assertTrue("dhtEntry=" + dhtEntry, dhtEntry.localCandidates().isEmpty()); - assertEquals(key, backupCache.localPeek(key, null, null)); + assertEquals(key, backupCache.localPeek(key, null)); if (nearEntry != null) { assertTrue("near=" + nearEntry, nearEntry.remoteMvccSnapshot().isEmpty()); assertTrue("near=" + nearEntry, nearEntry.localCandidates().isEmpty()); // Near peek wil be null since primary node has changed. - assertNull("near=" + nearEntry, origCache.localPeek(key, null, null)); + assertNull("near=" + nearEntry, origCache.localPeek(key, null)); } } else { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java index c89372ea15f58..63bfd41e3731d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMultiNodeSelfTest.java @@ -372,7 +372,7 @@ public void testReadThrough() throws Exception { GridCacheAdapter dhtCache = dht(G.ignite(n.id())); - String s = dhtCache.localPeek(key, null, null); + String s = dhtCache.localPeek(key, null); assert s != null : "Value is null for key: " + key; assertEquals(s, Integer.toString(key)); @@ -532,7 +532,7 @@ public void testPessimisticWriteThrough() throws Exception { assertEquals("2", near.get(2)); assertEquals("3", near.get(3)); - assertNotNull(dht(primaryGrid(3)).localPeek(3, null, null)); + assertNotNull(dht(primaryGrid(3)).localPeek(3, null)); tx.commit(); } @@ -961,4 +961,4 @@ boolean isEmpty() { map.remove(key); } } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java index 0ba3bcb221393..33ea8f8b52fd3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java @@ -17,6 +17,17 @@ package org.apache.ignite.internal.processors.igfs; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.Cache; import org.apache.commons.io.IOUtils; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; @@ -47,18 +58,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import javax.cache.Cache; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; - import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -219,7 +218,7 @@ public void testCreate() throws Exception { for (int i = 0; i < nodesCount(); i++) { IgfsEntryInfo fileInfo = - (IgfsEntryInfo)grid(i).cachex(metaCacheName).localPeek(info.fileId(), null, null); + (IgfsEntryInfo)grid(i).cachex(metaCacheName).localPeek(info.fileId(), null); assertNotNull(fileInfo); assertNotNull(fileInfo.listing()); @@ -989,4 +988,4 @@ private void assertListDir(String path, String... item) { assertEquals(Arrays.asList(item), names); } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 43a9e5c817835..647e578dd3856 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -1384,7 +1384,7 @@ protected static V dhtPeek(IgniteCache cache, K key) throws IgniteC * @param key Key. */ protected static V localPeek(GridCacheAdapter cache, K key) throws IgniteCheckedException { - return cache.localPeek(key, null, null); + return cache.localPeek(key, null); } /** @@ -1392,7 +1392,7 @@ protected static V localPeek(GridCacheAdapter cache, K key) throws * @param key Key. */ protected static V localPeekOnHeap(GridCacheAdapter cache, K key) throws IgniteCheckedException { - return cache.localPeek(key, new CachePeekMode[] {CachePeekMode.ONHEAP}, null); + return cache.localPeek(key, new CachePeekMode[] {CachePeekMode.ONHEAP}); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java index 930706d497b45..bcbc7f0e5f340 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java @@ -18,9 +18,6 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; -import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.internal.processors.cache.IgniteCacheTxPeekModesTest; -import org.apache.ignite.internal.processors.cache.IgniteCacheTxReplicatedPeekModesTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccClusterRestartTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccConfigurationValidationTest; import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccIteratorWithConcurrentTransactionTest; @@ -41,8 +38,6 @@ import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccPersistenceSelfTest; import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; - /** * */ @@ -69,9 +64,6 @@ public static TestSuite suite() { suite.addTestSuite(MvccCachePeekTest.class); - suite.addTestSuite(MvccIgniteCacheTxPeekModesTest.class); - suite.addTestSuite(MvccIgniteCacheTxReplicatedPeekModesTest.class); - // Concurrent ops tests. suite.addTestSuite(CacheMvccIteratorWithConcurrentTransactionTest.class); suite.addTestSuite(CacheMvccLocalEntriesWithConcurrentTransactionTest.class); @@ -87,34 +79,4 @@ public static TestSuite suite() { return suite; } - - /** */ - public static class MvccIgniteCacheTxPeekModesTest extends IgniteCacheTxPeekModesTest { - /** {@inheritDoc} */ - @Override protected CacheAtomicityMode atomicityMode() { - return TRANSACTIONAL_SNAPSHOT; - } - - /** {@inheritDoc} */ - @Override public void testLocalEntries() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10167"); - - super.testLocalEntries(); - } - } - - /** */ - public static class MvccIgniteCacheTxReplicatedPeekModesTest extends IgniteCacheTxReplicatedPeekModesTest { - /** {@inheritDoc} */ - @Override protected CacheAtomicityMode atomicityMode() { - return TRANSACTIONAL_SNAPSHOT; - } - - /** {@inheritDoc} */ - @Override public void testLocalEntries() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10167"); - - super.testLocalEntries(); - } - } } diff --git a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java index 0b659188941f2..d4efb5fd697c9 100644 --- a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java +++ b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java @@ -39,7 +39,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryPredicate; import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.lang.IgniteBiPredicate; @@ -124,10 +123,9 @@ public HibernateKeyTransformer keyTransformer(){ /** {@inheritDoc} */ @Nullable @Override public Object localPeek( Object key, - CachePeekMode[] peekModes, - @Nullable IgniteCacheExpiryPolicy plc + CachePeekMode[] peekModes ) throws IgniteCheckedException { - return delegate.localPeek(keyTransformer.transform(key), peekModes, plc); + return delegate.localPeek(keyTransformer.transform(key), peekModes); } /** {@inheritDoc} */ From bd2f87876a2866704aa9fc7f53f05a00931036cb Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Mon, 10 Dec 2018 14:49:21 +0300 Subject: [PATCH 233/403] IGNITE-10263: MVCC: Concurrent cache stop can cause vacuum failure. This closes #5409. (cherry picked from commit 2bf36865feea907243709b4e668e21aa7f8dd2ff) --- .../internal/processors/cache/mvcc/MvccProcessorImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java index 9ef0923ba4712..0a07df1d40d7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java @@ -2180,7 +2180,8 @@ private static class VacuumScheduler extends GridWorker { if (e instanceof Error) throw (Error) e; - U.error(log, "Vacuum error.", e); + if (log.isDebugEnabled()) + U.warn(log, "Failed to perform vacuum.", e); } long delay = nextScheduledTime - U.currentTimeMillis(); @@ -2439,9 +2440,8 @@ private void cleanup(GridDhtLocalPartition part, KeyCacheObject key, List) rest)) { + for (MvccDataRow row : ((List) rest)) part.dataStore().updateTxState(cctx, row); - } } else part.dataStore().updateTxState(cctx, (MvccDataRow) rest); } From 8292aa582977c1d1ab9d6665655e04a3db84fa75 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Tue, 11 Dec 2018 14:19:54 +0300 Subject: [PATCH 234/403] IGNITE-10472: MVCC: EntryProcessor resource injection. This closes #5622. (cherry picked from commit 7633e18f1b8b6fce9aaf780c428193f91a019008) --- .../processors/cache/IgniteCacheOffheapManagerImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 2f632f1e4a874..6df2f8af54771 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -1991,6 +1991,8 @@ else if (res == ResultType.VERSION_FOUND || // exceptional case assert res == ResultType.PREV_NULL; if (entryProc != null) { + entryProc = EntryProcessorResourceInjectorProxy.wrap(cctx.kernalContext(), entryProc); + CacheInvokeEntry.Operation op = applyEntryProcessor(cctx, key, ver, entryProc, invokeArgs, updateRow, oldRow); if (op == CacheInvokeEntry.Operation.NONE) { From 625f7dc29601660546d8ce07bc8fe418493a6c76 Mon Sep 17 00:00:00 2001 From: Roman Guseinov Date: Thu, 13 Dec 2018 11:53:27 +0800 Subject: [PATCH 235/403] IGNITE-9826 Prevent potential hang in S3 IP Finder - Fixes #5633. (cherry picked from commit 57c621cf1fdd0ba871b4cecfe439f59da13e1778) --- .../ignite/spi/discovery/tcp/ServerImpl.java | 6 +- .../tcp/TcpDiscoveryIpFinderCleanerTest.java | 163 ++++++++++++++++++ .../IgniteSpiDiscoverySelfTestSuite.java | 2 + 3 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIpFinderCleanerTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index a643b44282c92..141e2256ad89d 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -1962,9 +1962,6 @@ private IpFinderCleaner() { while (!isInterrupted()) { Thread.sleep(spi.ipFinderCleanFreq); - if (!isLocalNodeCoordinator()) - continue; - if (spiStateCopy() != CONNECTED) { if (log.isDebugEnabled()) log.debug("Stopping IP finder cleaner (SPI is not connected to topology)."); @@ -1972,6 +1969,9 @@ private IpFinderCleaner() { return; } + if (!isLocalNodeCoordinator()) + continue; + if (spi.ipFinder.isShared()) cleanIpFinder(); } diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIpFinderCleanerTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIpFinderCleanerTest.java new file mode 100644 index 0000000000000..52e3cf2aa5232 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryIpFinderCleanerTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.tcp; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests IP finder cleaner. + */ +public class TcpDiscoveryIpFinderCleanerTest extends GridCommonAbstractTest { + /** */ + private static final long IP_FINDER_CLEAN_FREQ = 1000; + + /** */ + private static final long NODE_STOPPING_TIMEOUT = 20000; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * Checks the node stops gracefully even if {@link TcpDiscoveryIpFinder} ignores {@link InterruptedException}. + * + * @throws Exception If failed. + */ + public void testNodeStops() throws Exception { + CustomIpFinder ipFinder = new CustomIpFinder(true); + + Ignite ignite = Ignition.start(getConfiguration(ipFinder)); + + try { + if (!ipFinder.suspend().await(IP_FINDER_CLEAN_FREQ * 5, TimeUnit.MILLISECONDS)) + fail("Failed to suspend IP finder."); + + if (!stopNodeAsync(ignite).await(NODE_STOPPING_TIMEOUT, TimeUnit.MILLISECONDS)) + fail("Node was not stopped."); + } + finally { + ipFinder.interruptCleanerThread(); + } + } + + /** + * @param ipFinder IP finder. + * @return Grid test configuration. + * @throws Exception If failed. + */ + private IgniteConfiguration getConfiguration(TcpDiscoveryIpFinder ipFinder) throws Exception { + TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi() + .setIpFinder(ipFinder) + .setIpFinderCleanFrequency(IP_FINDER_CLEAN_FREQ); + + return getConfiguration() + .setDiscoverySpi(discoverySpi); + } + + /** + * Stop the node asynchronously. + * + * @param node Ignite instance. + * @return Latch to signal when the node is stopped completely. + */ + private static CountDownLatch stopNodeAsync(final Ignite node) { + final CountDownLatch latch = new CountDownLatch(1); + + GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + try { + node.close(); + } + finally { + latch.countDown(); + } + } + }); + + return latch; + } + + /** + * Custom IP finder. + */ + private static class CustomIpFinder extends TcpDiscoveryVmIpFinder { + /** */ + private volatile boolean suspendFinderAndResetInterruptedFlag; + + /** */ + private final CountDownLatch suspended = new CountDownLatch(1); + + /** */ + private volatile Thread cleanerThread; + + /** {@inheritDoc} */ + public CustomIpFinder(boolean shared) { + super(shared); + } + + /** {@inheritDoc} */ + @Override public synchronized Collection getRegisteredAddresses() { + if (suspendFinderAndResetInterruptedFlag) { + cleanerThread = Thread.currentThread(); + + suspended.countDown(); + + try { + new CountDownLatch(1).await(); + } + catch (InterruptedException ignore) { + suspendFinderAndResetInterruptedFlag = false; + } + } + + return super.getRegisteredAddresses(); + } + + /** + * Suspend IP finder in {@link CustomIpFinder#getRegisteredAddresses()} method. + * + * @return Latch to signal when IP finder is suspended. + */ + public CountDownLatch suspend() { + suspendFinderAndResetInterruptedFlag = true; + + return suspended; + } + + /** + * Interrupt IP finder cleaner thread. + */ + public void interruptCleanerThread() { + if (cleanerThread != null) + cleanerThread.interrupt(); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java index 3ef1087a81efd..7622779f3f120 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java @@ -34,6 +34,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryUnresolvedHostTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryClientSuspensionSelfTest; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryIpFinderCleanerTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest; @@ -83,6 +84,7 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(TcpDiscoverySharedFsIpFinderSelfTest.class)); suite.addTest(new TestSuite(TcpDiscoveryJdbcIpFinderSelfTest.class)); suite.addTest(new TestSuite(TcpDiscoveryMulticastIpFinderSelfTest.class)); + suite.addTest(new TestSuite(TcpDiscoveryIpFinderCleanerTest.class)); suite.addTest(new TestSuite(TcpDiscoverySelfTest.class)); suite.addTest(new TestSuite(TcpDiscoverySpiSelfTest.class)); From 48913f1dc9f58804c2bc0396d7696c27da71791e Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Thu, 13 Dec 2018 10:50:39 +0700 Subject: [PATCH 236/403] IGNITE-10617 Web console: Fixed wrong class name. (cherry picked from commit c12f1c5192d4bc77b7c14e2b5ae4d5011c12aea2) --- .../connected-clusters-dialog/components/list/controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js index 2d0ca266b9a2e..b69f8a24c3924 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js +++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js @@ -19,7 +19,7 @@ import _ from 'lodash'; import columnDefs from './column-defs'; -export default class SnapshotsListCachesCtrl { +export default class ConnectedClustersListCtrl { static $inject = ['$scope', 'AgentManager']; /** From 0c334b49b0854331f15e2d6ad48b1001283f95af Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Thu, 13 Dec 2018 16:47:42 +0700 Subject: [PATCH 237/403] IGNITE-10667 Web Console: Fixed model for "keyStoreFilePath" on "Configuration" screen. (cherry picked from commit eb0584b0d394010fe6202579d33d36607d2a15bf) --- .../components/cluster-edit-form/templates/ssl.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug index 61c722ee2d7a5..f9791243e1fb4 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug @@ -51,7 +51,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`) .pc-form-grid-col-60 +form-field__text({ label: 'Key store file:', - model: `${secondaryFileSystem}.keyStoreFilePath`, + model: `${model}.keyStoreFilePath`, name: '"keyStoreFilePath"', disabled: `!(${enabled})`, required: enabled, From ed8dd109d8797e9437a90f379e90ae2ece9a706c Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Fri, 7 Dec 2018 18:56:48 +0300 Subject: [PATCH 238/403] ignite-10348 Safely recreate metastore to mitigate IGNITE-8735 --- .../internal/pagemem/PageIdAllocator.java | 5 +- .../processors/cache/GridCacheProcessor.java | 18 + .../file/FilePageStoreManager.java | 2 +- .../persistence/metastorage/MetaStorage.java | 407 +++++++++++++++++- .../metastorage/MetastorageDataRow.java | 3 +- .../metastorage/MetastorageTree.java | 11 +- .../IgnitePdsCorruptedStoreTest.java | 3 +- .../IgniteMetaStorageBasicTest.java | 273 +++++++++++- 8 files changed, 699 insertions(+), 23 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java index b3e4b07b9fb67..d91d31da32957 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java @@ -35,8 +35,11 @@ public interface PageIdAllocator { /** Special partition reserved for index space. */ public static final int INDEX_PARTITION = 0xFFFF; + /** Old special partition reserved for metastore space. */ + public static final int OLD_METASTORE_PARTITION = 0x0; + /** Special partition reserved for metastore space. */ - public static final int METASTORE_PARTITION = 0x0; + public static final int METASTORE_PARTITION = 0x1; /** * Allocates a page from the space for the given partition ID and the given flags. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 81285c8917826..ef4938157d247 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -106,6 +106,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage; import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; @@ -282,6 +283,9 @@ public class GridCacheProcessor extends GridProcessorAdapter { /** Cache recovery lifecycle state and actions. */ private final CacheRecoveryLifecycle recovery = new CacheRecoveryLifecycle(); + /** Tmp storage for meta migration. */ + private MetaStorage.TmpStorage tmpStorage; + /** * @param ctx Kernal context. */ @@ -5478,6 +5482,20 @@ public T clone(final T obj) throws IgniteCheckedException { }); } + /** + * Get Temporary storage + */ + public MetaStorage.TmpStorage getTmpStorage() { + return tmpStorage; + } + + /** + * Set Temporary storage + */ + public void setTmpStorage(MetaStorage.TmpStorage tmpStorage) { + this.tmpStorage = tmpStorage; + } + /** * Recovery lifecycle for caches. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 4a14c6b261018..144f7f79c6b1a 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -361,7 +361,7 @@ public FilePageStoreManager(GridKernalContext ctx) { CacheStoreHolder holder = initDir( new File(storeWorkDir, META_STORAGE_NAME), grpId, - 1, + PageIdAllocator.METASTORE_PARTITION + 1, AllocatedPageTracker.NO_OP, false); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java index 91fbec096029d..a444909d2eb36 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java @@ -17,13 +17,24 @@ package org.apache.ignite.internal.processors.cache.persistence.metastorage; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; import java.io.Serializable; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; @@ -32,10 +43,11 @@ import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; -import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.IncompleteObject; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; @@ -44,6 +56,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.RootPage; +import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO; @@ -58,12 +71,14 @@ import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.marshaller.Marshaller; import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.jetbrains.annotations.NotNull; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.OLD_METASTORE_PARTITION; import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId; import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId; @@ -77,9 +92,19 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R /** */ public static final int METASTORAGE_CACHE_ID = CU.cacheId(METASTORAGE_CACHE_NAME); + /** This flag is used ONLY FOR TESTING the migration of a metastorage from Part 0 to Part 1. */ + public static boolean PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false; + /** Marker for removed entry. */ private static final byte[] TOMBSTONE = new byte[0]; + /** Temporary metastorage memory size. */ + private static final int TEMPORARY_METASTORAGE_IN_MEMORY_SIZE = 128 * 1024 * 1024; + + /** Temporary metastorage buffer size (file). */ + private static final int TEMPORARY_METASTORAGE_BUFFER_SIZE = 1024 * 1024; + + /** */ private final IgniteWriteAheadLogManager wal; @@ -99,7 +124,7 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R private DataRegionMetricsImpl regionMetrics; /** */ - private boolean readOnly; + private final boolean readOnly; /** */ private boolean empty; @@ -122,6 +147,12 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R /** */ private final FailureProcessor failureProcessor; + /** Partition id. */ + private int partId; + + /** Cctx. */ + private final GridCacheSharedContext cctx; + /** */ public MetaStorage( GridCacheSharedContext cctx, @@ -129,6 +160,7 @@ public MetaStorage( DataRegionMetricsImpl regionMetrics, boolean readOnly ) { + this.cctx = cctx; wal = cctx.wal(); this.dataRegion = dataRegion; this.regionMetrics = regionMetrics; @@ -138,8 +170,78 @@ public MetaStorage( } /** */ - public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException { - getOrAllocateMetas(); + public void init(GridCacheDatabaseSharedManager db) throws IgniteCheckedException { + initInternal(db); + + if (!PRESERVE_LEGACY_METASTORAGE_PARTITION_ID) { + GridCacheProcessor gcProcessor = cctx.kernalContext().cache(); + + if (partId == OLD_METASTORE_PARTITION) + gcProcessor.setTmpStorage(copyDataToTmpStorage()); + else if (gcProcessor.getTmpStorage() != null) { + restoreDataFromTmpStorage(gcProcessor.getTmpStorage()); + + gcProcessor.setTmpStorage(null); + + // remove old partitions + CacheGroupContext cgc = cctx.cache().cacheGroup(METASTORAGE_CACHE_ID); + + if (cgc != null) { + db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION); + + db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION); + } + } + } + } + + /** + * Copying all data from the 'meta' to temporary storage. + * + * @return Target temporary storage + */ + private TmpStorage copyDataToTmpStorage() throws IgniteCheckedException { + TmpStorage tmpStorage = new TmpStorage(TEMPORARY_METASTORAGE_IN_MEMORY_SIZE, log); + + GridCursor cur = tree.find(null, null); + + while (cur.next()) { + MetastorageDataRow row = cur.get(); + + tmpStorage.add(row.key(), row.value()); + } + + return tmpStorage; + } + + /** + * Data recovery from temporary storage + * + * @param tmpStorage temporary storage. + */ + private void restoreDataFromTmpStorage(TmpStorage tmpStorage) throws IgniteCheckedException { + for (Iterator> it = tmpStorage.stream().iterator(); it.hasNext(); ) { + IgniteBiTuple t = it.next(); + + putData(t.get1(), t.get2()); + } + + try { + tmpStorage.close(); + } + catch (IOException e) { + log.error(e.getMessage(), e); + } + } + + /** + * @param db Database. + */ + private void initInternal(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException { + if (PRESERVE_LEGACY_METASTORAGE_PARTITION_ID) + getOrAllocateMetas(partId = PageIdAllocator.OLD_METASTORE_PARTITION); + else if (!readOnly || getOrAllocateMetas(partId = PageIdAllocator.OLD_METASTORE_PARTITION)) + getOrAllocateMetas(partId = PageIdAllocator.METASTORE_PARTITION); if (!empty) { freeList = new FreeListImpl(METASTORAGE_CACHE_ID, "metastorage", @@ -149,7 +251,7 @@ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedExcept MetastorageRowStore rowStore = new MetastorageRowStore(freeList, db); tree = new MetastorageTree(METASTORAGE_CACHE_ID, dataRegion.pageMemory(), wal, rmvId, - freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated(), failureProcessor); + freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated(), failureProcessor, partId); if (!readOnly) ((GridCacheDatabaseSharedManager)db).addCheckpointListener(this); @@ -224,6 +326,23 @@ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedExcept return res; } + /** + * Read all items from metastore. + */ + public Collection> readAll() throws IgniteCheckedException { + ArrayList> res = new ArrayList<>(); + + GridCursor cur = tree.find(null, null); + + while (cur.next()) { + MetastorageDataRow row = cur.get(); + + res.add(new IgniteBiTuple<>(row.key(), row.value())); + } + + return res; + } + /** {@inheritDoc} */ @Override public void write(@NotNull String key, @NotNull Serializable val) throws IgniteCheckedException { assert val != null; @@ -312,11 +431,16 @@ private void checkRootsPageIdFlag(long treeRoot, long reuseListRoot) throws Stor + U.hexLong(reuseListRoot) + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID); } - /** */ - private void getOrAllocateMetas() throws IgniteCheckedException { - PageMemoryEx pageMem = (PageMemoryEx)dataRegion.pageMemory(); + /** + * Initializing the selected partition for use as MetaStorage + * + * @param partId Partition id. + * @return true if the partion is empty + */ + private boolean getOrAllocateMetas(int partId) throws IgniteCheckedException { + empty = false; - int partId = 0; + PageMemoryEx pageMem = (PageMemoryEx)dataRegion.pageMemory(); long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId); long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId); @@ -328,7 +452,7 @@ private void getOrAllocateMetas() throws IgniteCheckedException { if (PageIO.getType(pageAddr) != PageIO.T_PART_META) { empty = true; - return; + return true; } PagePartitionMetaIO io = PageIO.getPageIO(pageAddr); @@ -404,6 +528,8 @@ private void getOrAllocateMetas() throws IgniteCheckedException { finally { pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage); } + + return false; } /** @@ -449,8 +575,6 @@ public PageMemory pageMemory() { private void saveStoreMetadata() throws IgniteCheckedException { PageMemoryEx pageMem = (PageMemoryEx) pageMemory(); - int partId = 0; - long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId); long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId); @@ -496,7 +620,7 @@ public void applyUpdate(String key, byte[] value) throws IgniteCheckedException } /** */ - public static class FreeListImpl extends AbstractFreeList { + public class FreeListImpl extends AbstractFreeList { /** {@inheritDoc} */ FreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionMetrics, DataRegion dataRegion, ReuseList reuseList, @@ -511,7 +635,7 @@ public static class FreeListImpl extends AbstractFreeList { /** {@inheritDoc} */ @Override protected long allocatePageNoReuse() throws IgniteCheckedException { - return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA); + return pageMem.allocatePage(grpId, partId, FLAG_DATA); } /** @@ -596,4 +720,259 @@ final MetastorageDataRow readRow(String key, long link) return new MetastorageDataRow(link, key, incomplete.data()); } } + + /** + * Temporary storage internal + */ + private interface TmpStorageInternal extends Closeable { + /** + * Put data + * + * @param key Key. + * @param val Value. + */ + boolean add(String key, byte[] val) throws IOException; + + /** + * Read data from storage + */ + Stream> stream() throws IOException; + } + + /** + * Temporary storage (memory) + */ + private static class MemoryTmpStorage implements TmpStorageInternal { + /** Buffer. */ + final ByteBuffer buf; + + /** Size. */ + int size; + + /** + * @param size Size. + */ + MemoryTmpStorage(int size) { + buf = ByteBuffer.allocateDirect(size); + } + + /** {@inheritDoc} */ + @Override public boolean add(String key, byte[] val) { + byte[] keyData = key.getBytes(StandardCharsets.UTF_8); + + if (val.length + keyData.length + 8 > buf.remaining()) + return false; + + buf.putInt(keyData.length).putInt(val.length).put(keyData).put(val); + + size++; + + return true; + } + + /** {@inheritDoc} */ + @Override public Stream> stream() { + buf.flip(); + + return Stream.generate(() -> { + int keyLen = buf.getInt(); + int dataLen = buf.getInt(); + + byte[] tmpBuf = new byte[Math.max(keyLen, dataLen)]; + + buf.get(tmpBuf, 0, keyLen); + + String key = new String(tmpBuf, 0, keyLen, StandardCharsets.UTF_8); + + buf.get(tmpBuf, 0, dataLen); + + return new IgniteBiTuple<>(key, tmpBuf.length > dataLen ? Arrays.copyOf(tmpBuf, dataLen) : tmpBuf); + }).limit(size); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + } + } + + /** + * Temporary storage (file) + */ + private static class FileTmpStorage implements TmpStorageInternal { + /** Cache. */ + final ByteBuffer cache = ByteBuffer.allocateDirect(TEMPORARY_METASTORAGE_BUFFER_SIZE); + + /** File. */ + RandomAccessFile file; + + /** Size. */ + long size; + + /** {@inheritDoc} */ + @Override public boolean add(String key, byte[] val) throws IOException { + if (file == null) + file = new RandomAccessFile(File.createTempFile("m_storage", "bin"), "rw"); + + byte[] keyData = key.getBytes(StandardCharsets.UTF_8); + + if (val.length + keyData.length + 8 > cache.remaining()) + flushCache(false); + + cache.putInt(keyData.length).putInt(val.length).put(keyData).put(val); + + size++; + + return true; + } + + /** {@inheritDoc} */ + @Override public Stream> stream() throws IOException { + if (file == null) + return Stream.empty(); + + flushCache(true); + + file.getChannel().position(0); + + readToCache(); + + return Stream.generate(() -> { + if (cache.remaining() <= 8) { + cache.compact(); + + try { + readToCache(); + } + catch (IOException e) { + throw new IgniteException(e); + } + } + + int keyLen = cache.getInt(); + int dataLen = cache.getInt(); + + if (cache.remaining() < keyLen + dataLen) { + cache.compact(); + + try { + readToCache(); + } + catch (IOException e) { + throw new IgniteException(e); + } + } + + byte[] tmpBuf = new byte[Math.max(keyLen, dataLen)]; + + cache.get(tmpBuf, 0, keyLen); + + String key = new String(tmpBuf, 0, keyLen, StandardCharsets.UTF_8); + + cache.get(tmpBuf, 0, dataLen); + + return new IgniteBiTuple<>(key, tmpBuf.length > dataLen ? Arrays.copyOf(tmpBuf, dataLen) : tmpBuf); + }).limit(size); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + file.close(); + } + + /** + * Read data to cache + */ + private void readToCache() throws IOException { + int len = (int)Math.min(file.length() - file.getChannel().position(), cache.remaining()); + + while (len > 0) + len -= file.getChannel().read(cache); + + cache.flip(); + } + + /** + * Write cache to file. + * + * @param force force metadata. + */ + private void flushCache(boolean force) throws IOException { + if (cache.position() > 0) { + cache.flip(); + + while (cache.remaining() > 0) + file.getChannel().write(cache); + + cache.clear(); + } + + file.getChannel().force(force); + } + } + + /** + * Temporary storage + */ + public static class TmpStorage implements Closeable { + /** Chain of internal storages. */ + final List chain = new ArrayList<>(2); + + /** Current internal storage. */ + TmpStorageInternal current; + + /** Logger. */ + final IgniteLogger log; + + /** + * @param memBufSize Memory buffer size. + * @param log Logger. + */ + TmpStorage(int memBufSize, IgniteLogger log) { + this.log = log; + + chain.add(current = new MemoryTmpStorage(memBufSize)); + } + + /** + * Put data + * + * @param key Key. + * @param val Value. + */ + public void add(String key, byte[] val) throws IgniteCheckedException { + try { + while (!current.add(key, val)) + chain.add(current = new FileTmpStorage()); + } + catch (IOException e) { + throw new IgniteCheckedException(e); + } + } + + /** + * Read data from storage + */ + public Stream> stream() { + return chain.stream().flatMap(storage -> { + try { + return storage.stream(); + } + catch (IOException e) { + throw new IgniteException(e); + } + }); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + for (TmpStorageInternal storage : chain) { + try { + storage.close(); + } + catch (IOException ex) { + log.error(ex.getMessage(), ex); + } + } + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java index 5e2660bc3aa97..2d7b0a66e47a4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.persistence.metastorage; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.processors.cache.persistence.Storable; /** @@ -62,7 +63,7 @@ public int hash() { /** {@inheritDoc} */ @Override public int partition() { - return 0; + return MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID ? PageIdAllocator.OLD_METASTORE_PARTITION: PageIdAllocator.METASTORE_PARTITION; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java index 27de6841c9084..72e7f510c9f9c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java @@ -19,7 +19,6 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -45,6 +44,9 @@ public class MetastorageTree extends BPlusTree putDataToMetaStorage(MetaStorage metaStorage, int size, int from) throws IgniteCheckedException { + Map res = new HashMap<>(); + + for (Iterator> it = generateTestData(size, from).iterator(); it.hasNext(); ) { + IgniteBiTuple d = it.next(); + + metaStorage.putData(d.getKey(), d.getValue()); + + res.put(d.getKey(), d.getValue()); + } + + return res; + } + + /** + * Testing data migration between metastorage partitions (delete partition case) + */ + public void testDeletePartitionFromMetaStorageMigration() throws Exception { + final Map testData = new HashMap<>(); + + MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = true; + + try { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database(); + + MetaStorage metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + db.checkpointReadLock(); + + try { + testData.putAll(putDataToMetaStorage(metaStorage, 1_000, 0)); + } + finally { + db.checkpointReadUnlock(); + } + + db.waitForCheckpoint("Test"); + + ((GridCacheDatabaseSharedManager)db).enableCheckpoints(false); + + db.checkpointReadLock(); + + try { + testData.putAll(putDataToMetaStorage(metaStorage, 1_000, 1_000)); + } + finally { + db.checkpointReadUnlock(); + } + + stopGrid(0); + + MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false; + + IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(0)); + + cfg.getDataStorageConfiguration().setCheckpointFrequency(3600 * 1000L); + + ig = (IgniteEx)startGrid(getTestIgniteInstanceName(0), optimize(cfg), null); + + ig.cluster().active(true); + + db = ig.context().cache().context().database(); + + metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + db.checkpointReadLock(); + + try { + testData.putAll(putDataToMetaStorage(metaStorage, 1_000, 2_000)); + } + finally { + db.checkpointReadUnlock(); + } + + db.waitForCheckpoint("Test"); + + stopGrid(0); + + ig = startGrid(0); + + ig.cluster().active(true); + + db = ig.context().cache().context().database(); + + metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + db.checkpointReadLock(); + try { + Collection> read = metaStorage.readAll(); + + int cnt = 0; + for (IgniteBiTuple r : read) { + byte[] test = testData.get(r.get1()); + + if (test != null) { + Assert.assertArrayEquals(r.get2(), test); + + cnt++; + } + } + + assertEquals(cnt, testData.size()); + } + finally { + db.checkpointReadUnlock(); + } + } + finally { + MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false; + } + + } + + /** + * Testing data migration between metastorage partitions + */ + public void testMetaStorageMigration() throws Exception { + final Map testData = new HashMap<>(5_000); + + generateTestData(5_000, -1).forEach(t -> testData.put(t.get1(), t.get2())); + + MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = true; + + try { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database(); + + MetaStorage metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + db.checkpointReadLock(); + + try { + for (Map.Entry v : testData.entrySet()) + metaStorage.putData(v.getKey(), v.getValue()); + } + finally { + db.checkpointReadUnlock(); + } + + stopGrid(0); + + MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false; + + ig = startGrid(0); + + ig.cluster().active(true); + + db = ig.context().cache().context().database(); + + metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + db.checkpointReadLock(); + + try { + Collection> read = metaStorage.readAll(); + + int cnt = 0; + for (IgniteBiTuple r : read) { + byte[] test = testData.get(r.get1()); + + if (test != null) { + Assert.assertArrayEquals(r.get2(), test); + + cnt++; + } + } + + assertEquals(cnt, testData.size()); + } + finally { + db.checkpointReadUnlock(); + } + } + finally { + MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false; + } + } + + /** + * Testing temporary storage + */ + public void testMetaStoreMigrationTmpStorage() throws Exception { + List> data = generateTestData(2_000, -1).collect(Collectors.toList()); + + // memory + try (MetaStorage.TmpStorage tmpStorage = new MetaStorage.TmpStorage(4 * 1024 * 1024, log)) { + for (IgniteBiTuple item : data) + tmpStorage.add(item.get1(), item.get2()); + + compare(tmpStorage.stream().iterator(), data.iterator()); + } + + // file + try (MetaStorage.TmpStorage tmpStorage = new MetaStorage.TmpStorage(4 * 1024, log)) { + for (IgniteBiTuple item : data) + tmpStorage.add(item.get1(), item.get2()); + + compare(tmpStorage.stream().iterator(), data.iterator()); + } + } + + /** + * Test data generation + */ + private static Stream> generateTestData(int size, int fromKey) { + final AtomicInteger idx = new AtomicInteger(fromKey); + final Random rnd = new Random(); + + return Stream.generate(() -> { + byte[] val = new byte[1024]; + + rnd.nextBytes(val); + + return new IgniteBiTuple<>("KEY_" + (fromKey < 0 ? rnd.nextInt() : idx.getAndIncrement()), val); + }).limit(size); + } + + /** + * Compare two iterator + * + * @param it It. + * @param it1 It 1. + */ + private static void compare(Iterator> it, Iterator> it1) { + while (true) { + Assert.assertEquals(it.hasNext(), it1.hasNext()); + + if (!it.hasNext()) + break; + + IgniteBiTuple i = it.next(); + IgniteBiTuple i1 = it1.next(); + + Assert.assertEquals(i.get1(), i.get1()); + + Assert.assertArrayEquals(i.get2(), i1.get2()); + } } /** From 0c9c0fb240effbb6e7ec5f6e9b6803f4d62d9499 Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Thu, 13 Dec 2018 12:26:41 +0300 Subject: [PATCH 239/403] IGNITE-10446 Fix control.sh --cache idle_verify fail with NPE when node left grid Signed-off-by: Dmitriy Govorukhin (cherry picked from commit d7af985383d9e82608fd77904415960492ed6290) --- .../internal/commandline/CommandHandler.java | 4 +- .../cache/verify/IdleVerifyResultV2.java | 34 ++- .../VerifyBackupPartitionsDumpTask.java | 28 +++ .../verify/VerifyBackupPartitionsTaskV2.java | 31 ++- .../ignite/util/GridCommandHandlerTest.java | 197 +++++++++++++----- 5 files changed, 240 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index f6094c7527298..1bc7f3a57400c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -836,7 +836,7 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws log("Contention check failed on nodes:"); for (Map.Entry e : res.exceptions().entrySet()) { - log("Node ID = " + e.getKey()); + log("Node ID: " + e.getKey()); log("Exception message:"); log(e.getValue().getMessage()); @@ -871,7 +871,7 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t log("Index validation failed on nodes:"); for (Map.Entry e : taskRes.exceptions().entrySet()) { - log(i("Node ID = " + e.getKey())); + log(i("Node ID: " + e.getKey())); log(i("Exception message:")); log(i(e.getValue().getMessage(), 2)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java index c31d6627ed94a..a153063d8998a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java @@ -21,6 +21,7 @@ import java.io.ObjectOutput; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.function.Consumer; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; @@ -43,6 +44,9 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject { /** Moving partitions. */ private Map> movingPartitions; + /** Exceptions. */ + private Map exceptions; + /** * @param cntrConflicts Counter conflicts. * @param hashConflicts Hash conflicts. @@ -51,11 +55,13 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject { public IdleVerifyResultV2( Map> cntrConflicts, Map> hashConflicts, - Map> movingPartitions + Map> movingPartitions, + Map exceptions ) { this.cntrConflicts = cntrConflicts; this.hashConflicts = hashConflicts; this.movingPartitions = movingPartitions; + this.exceptions = exceptions; } /** @@ -64,11 +70,17 @@ public IdleVerifyResultV2( public IdleVerifyResultV2() { } + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V2; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeMap(out, cntrConflicts); U.writeMap(out, hashConflicts); U.writeMap(out, movingPartitions); + U.writeMap(out, exceptions); } /** {@inheritDoc} */ @@ -77,6 +89,9 @@ public IdleVerifyResultV2() { cntrConflicts = U.readMap(in); hashConflicts = U.readMap(in); movingPartitions = U.readMap(in); + + if (protoVer >= V2) + exceptions = U.readMap(in); } /** @@ -107,6 +122,13 @@ public boolean hasConflicts() { return !F.isEmpty(hashConflicts()) || !F.isEmpty(counterConflicts()); } + /** + * @return Exceptions on nodes. + */ + public Map exceptions() { + return exceptions; + } + /** * Print formatted result to given printer. * @@ -159,6 +181,16 @@ public void print(Consumer printer) { printer.accept("\n"); } + + if (!F.isEmpty(exceptions())) { + printer.accept("Idle verify failed on nodes:\n"); + + for (Map.Entry e : exceptions().entrySet()) { + printer.accept("Node ID: " + e.getKey() + "\n"); + printer.accept("Exception message:" + "\n"); + printer.accept(e.getValue().getMessage() + "\n"); + } + } } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java index 7bbd9f19b9b15..5945f4d0990b1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java @@ -30,15 +30,18 @@ import java.util.TreeMap; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.resources.LoggerResource; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -68,6 +71,10 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter map( List subgrid, VisorIdleVerifyTaskArg arg) throws IgniteException { @@ -83,6 +90,9 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter> clusterHashes = new TreeMap<>(buildPartitionKeyComparator()); for (ComputeJobResult res : results) { + if (res.getException() != null) + continue; + Map nodeHashes = res.getData(); for (Map.Entry e : nodeHashes.entrySet()) { @@ -111,6 +121,22 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter rcvd) throws + IgniteException { + ComputeJobResultPolicy superRes = super.result(res, rcvd); + + // Deny failover. + if (superRes == ComputeJobResultPolicy.FAILOVER) { + superRes = ComputeJobResultPolicy.WAIT; + + log.warning("VerifyBackupPartitionsJobV2 failed on node " + + "[consistentId=" + res.getNode().consistentId() + "]", res.getException()); + } + + return superRes; + } + /** * Checking conditions for adding given record to result. * @@ -140,6 +166,8 @@ record = records.get(i); /** * @param partitions Dump result. + * @param conflictRes Conflict results. + * @param skippedRecords Number of empty partitions. * @return Path where results are written. * @throws IgniteException If failed to write the file. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index e698e3a2e6bc4..acd8170c48b3e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; @@ -40,6 +41,7 @@ import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobAdapter; import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.CacheGroupContext; @@ -71,6 +73,10 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter results) throws IgniteException { Map> clusterHashes = new HashMap<>(); + Map exceptions = new HashMap<>(); for (ComputeJobResult res : results) { + if (res.getException() != null) { + exceptions.put(res.getNode().id(), res.getException()); + + continue; + } + Map nodeHashes = res.getData(); for (Map.Entry e : nodeHashes.entrySet()) { @@ -135,7 +148,23 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter rcvd) throws + IgniteException { + ComputeJobResultPolicy superRes = super.result(res, rcvd); + + // Deny failover. + if (superRes == ComputeJobResultPolicy.FAILOVER) { + superRes = ComputeJobResultPolicy.WAIT; + + log.warning("VerifyBackupPartitionsJobV2 failed on node " + + "[consistentId=" + res.getNode().consistentId() + "]", res.getException()); + } + + return superRes; } /** diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6edde596e6da3..5448162ef588d 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; import java.io.PrintStream; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -46,6 +48,7 @@ import org.apache.ignite.IgniteAtomicSequence; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; @@ -54,6 +57,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.GridJobExecuteResponse; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; @@ -885,13 +889,7 @@ public void testCacheIdleVerify() throws Exception { ignite.cluster().active(true); - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -918,15 +916,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception { ignite.cluster().active(true); - int parts = 32; - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -938,7 +928,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception { corruptDataEntry(cacheCtx, 1, true, false); - corruptDataEntry(cacheCtx, 1 + parts / 2, false, true); + corruptDataEntry(cacheCtx, 1 + cacheCtx.config().getAffinity().partitions() / 2, false, true); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify")); @@ -955,12 +945,11 @@ public void testCacheIdleVerifyDump() throws Exception { ignite.cluster().active(true); - int parts = 32; + int keysCount = 20;//less than parts number for ability to check skipZeros flag. - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); + createCacheAndPreload(ignite, keysCount); + + int parts = ignite.affinity(DEFAULT_CACHE_NAME).partitions(); ignite.createCache(new CacheConfiguration<>() .setAffinity(new RendezvousAffinityFunction(false, parts)) @@ -969,11 +958,6 @@ public void testCacheIdleVerifyDump() throws Exception { injectTestSystemOut(); - int keysCount = 20;//less than parts number for ability to check skipZeros flag. - - for (int i = 0; i < keysCount; i++) - cache.put(i, i); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME)); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME)); @@ -1041,23 +1025,24 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception { ignite.cluster().active(true); - int parts = 32; - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); - for (int i = 0; i < 100; i++) - cache.put(i, i); + corruptingAndCheckDefaultCache(ignite); + } + + /** + * @param ignite Ignite. + */ + private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException { + injectTestSystemOut(); GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); corruptDataEntry(cacheCtx, 0, true, false); - corruptDataEntry(cacheCtx, 0 + parts / 2, false, true); + corruptDataEntry(cacheCtx, cacheCtx.config().getAffinity().partitions() / 2, false, true); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); @@ -1072,6 +1057,130 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception { fail("Should be found dump with conflicts"); } + /** + * Tests that idle verify print partitions info when node failing. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpWhenNodeFailing() throws Exception { + Ignite ignite = startGrids(3); + + Ignite unstable = startGrid("unstable"); + + ignite.cluster().active(true); + + createCacheAndPreload(ignite, 100); + + for (int i = 0; i < 3; i++) { + TestRecordingCommunicationSpi.spi(unstable).blockMessages(GridJobExecuteResponse.class, + getTestIgniteInstanceName(i)); + } + + injectTestSystemOut(); + + IgniteInternalFuture fut = GridTestUtils.runAsync(() -> { + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); + }); + + TestRecordingCommunicationSpi.spi(unstable).waitForBlocked(); + + UUID unstableNodeId = unstable.cluster().localNode().id(); + + unstable.close(); + + fut.get(); + + checkExceptionMessageOnReport(unstableNodeId); + } + + /** + * Tests that idle verify print partitions info when several nodes failing at same time. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpWhenSeveralNodesFailing() throws Exception { + int nodes = 6; + + Ignite ignite = startGrids(nodes); + + List unstableNodes = new ArrayList<>(nodes / 2); + + for (int i = 0; i < nodes; i++) { + if (i % 2 == 1) + unstableNodes.add(ignite(i)); + } + + ignite.cluster().active(true); + + createCacheAndPreload(ignite, 100); + + for (Ignite unstable : unstableNodes) { + for (int i = 0; i < nodes; i++) { + TestRecordingCommunicationSpi.spi(unstable).blockMessages(GridJobExecuteResponse.class, + getTestIgniteInstanceName(i)); + } + } + + injectTestSystemOut(); + + IgniteInternalFuture fut = GridTestUtils.runAsync(() -> { + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); + }); + + List unstableNodeIds = new ArrayList<>(nodes / 2); + + for (Ignite unstable : unstableNodes) { + TestRecordingCommunicationSpi.spi(unstable).waitForBlocked(); + + unstableNodeIds.add(unstable.cluster().localNode().id()); + + unstable.close(); + } + + fut.get(); + + for (UUID unstableId : unstableNodeIds) + checkExceptionMessageOnReport(unstableId); + } + + /** + * Creates default cache and preload some data entries. + * + * @param ignite Ignite. + * @param countEntries Count of entries. + */ + private void createCacheAndPreload(Ignite ignite, int countEntries) { + ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1)); + + try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < countEntries; i++) + streamer.addData(i, i); + } + } + + /** + * Try to finds node failed exception message on output report. + * + * @param unstableNodeId Unstable node id. + */ + private void checkExceptionMessageOnReport(UUID unstableNodeId) throws IOException { + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("Idle verify failed on nodes:")); + + assertTrue(dumpWithConflicts.contains("Node ID: " + unstableNodeId + "\n" + + "Exception message:\n" + + "Node has left grid: " + unstableNodeId)); + } + else + fail("Should be found dump with conflicts"); + } + /** * @return Build matcher for dump file name. */ @@ -1370,13 +1479,7 @@ public void testCacheDistribution() throws Exception { ignite.cluster().active(true); - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -1420,13 +1523,7 @@ public void testCacheResetLostPartitions() throws Exception { ignite.cluster().active(true); - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); From cb5c840ac884c96f78fdb6f68e7ba574b3ab59e4 Mon Sep 17 00:00:00 2001 From: Sergey Kosarev Date: Thu, 13 Dec 2018 15:53:56 +0300 Subject: [PATCH 240/403] IGNITE-10659 Fixed deadlock when binary metadata is requested for toString() - Fixes #5652. Signed-off-by: Alexey Goncharuk (cherry picked from commit a16eb0b0d418b8a5ec0df644cbe4cb744eafbbb9) --- .../processors/cache/binary/CacheObjectBinaryProcessorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index 4579636d7e8fd..498a6700215d9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -576,7 +576,7 @@ public GridBinaryMarshaller marshaller() { } if (holder != null) { - if (curThread instanceof IgniteDiscoveryThread) + if (curThread instanceof IgniteDiscoveryThread || (curThread != null && curThread.isForbiddenToRequestBinaryMetadata())) return holder.metadata(); if (holder.pendingVersion() - holder.acceptedVersion() > 0) { From 4ecc1ff54c559a159469a9004d927f261b217508 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 12 Dec 2018 14:07:52 +0700 Subject: [PATCH 241/403] IGNITE-10189 Implemented support for multiple protocols and cipher suites for utilities that work via o.a.i.internal.client.GridClient. (cherry picked from commit 9098813d6cbaa354f9ff1fef3dc4f17a499aa4d5) --- .../client/ClientSslParametersTest.java | 335 ++++++++++++++++++ .../client/suite/IgniteClientTestSuite.java | 4 + .../ssl/GridSslBasicContextFactory.java | 88 ++++- .../internal/commandline/Arguments.java | 38 +- .../internal/commandline/CommandHandler.java | 110 ++++-- .../apache/ignite/ssl/SSLContextWrapper.java | 13 +- .../apache/ignite/ssl/SslContextFactory.java | 16 +- .../ignite/client/SslParametersTest.java | 163 +++++---- .../util/GridCommandHandlerSslTest.java | 107 +++++- 9 files changed, 726 insertions(+), 148 deletions(-) create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java new file mode 100644 index 0000000000000..d865e5d8d028f --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.client; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.ConnectorConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.ssl.SslContextFactory; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.NotNull; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests cases when node connects to cluster with different set of cipher suites. + */ +@RunWith(JUnit4.class) +public class ClientSslParametersTest extends GridCommonAbstractTest { + /** */ + public static final String TEST_CACHE_NAME = "TEST"; + + /** */ + private volatile String[] cipherSuites; + + /** */ + private volatile String[] protocols; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + cfg.setSslContextFactory(createSslFactory()); + + cfg.setConnectorConfiguration(new ConnectorConfiguration() + .setSslEnabled(true) + .setSslClientAuth(true)); + + cfg.setCacheConfiguration(new CacheConfiguration(TEST_CACHE_NAME)); + + return cfg; + } + + /** + * @return Client configuration. + */ + protected GridClientConfiguration getClientConfiguration() { + GridClientConfiguration cfg = new GridClientConfiguration(); + + cfg.setServers(Collections.singleton("127.0.0.1:11211")); + + cfg.setSslContextFactory(createOldSslFactory()); + + return cfg; + } + + /** + * @return SSL factory. + */ + @NotNull private SslContextFactory createSslFactory() { + SslContextFactory factory = (SslContextFactory)GridTestUtils.sslFactory(); + + factory.setCipherSuites(cipherSuites); + + factory.setProtocols(protocols); + + return factory; + } + + /** + * @return SSL Factory. + */ + @NotNull private GridSslBasicContextFactory createOldSslFactory() { + GridSslBasicContextFactory factory = (GridSslBasicContextFactory)GridTestUtils.sslContextFactory(); + + factory.setCipherSuites(cipherSuites); + + factory.setProtocols(protocols); + + return factory; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + protocols = null; + + cipherSuites = null; + } + + /** + * @throws Exception If failed. + */ + @Test + public void testSameCipherSuite() throws Exception { + cipherSuites = new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" + }; + + startGrid(); + + checkSuccessfulClientStart( + new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" + }, + null + ); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testOneCommonCipherSuite() throws Exception { + cipherSuites = new String[] { + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" + }; + + startGrid(); + + checkSuccessfulClientStart( + new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" + }, + null + ); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testNoCommonCipherSuite() throws Exception { + cipherSuites = new String[] { + "TLS_RSA_WITH_AES_128_GCM_SHA256" + }; + + startGrid(); + + checkClientStartFailure( + new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" + }, + null + ); + } + + /** + * @throws Exception If failed. + */ + @Test + @Ignore("https://issues.apache.org/jira/browse/IGNITE-10245") + public void testNonExistentCipherSuite() throws Exception { + cipherSuites = new String[] { + "TLS_RSA_WITH_AES_128_GCM_SHA256" + }; + + startGrid(); + + checkClientStartFailure( + new String[] { + "TLC_FAKE_CIPHER", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" + }, + null, + "Unsupported ciphersuite" + ); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testNoCommonProtocols() throws Exception { + protocols = new String[] { + "TLSv1.1", + "SSLv3" + }; + + startGrid(); + + checkClientStartFailure( + null, + new String[] { + "TLSv1", + "TLSv1.2" + } + ); + } + + /** + * @throws Exception If failed. + */ + @Test + @Ignore("https://issues.apache.org/jira/browse/IGNITE-10245") + public void testNonExistentProtocol() throws Exception { + protocols = new String[] { + "SSLv3" + }; + + startGrid(); + + checkClientStartFailure( + null, + new String[] { + "SSLv3", + "SSLvDoesNotExist" + }, + "SSLvDoesNotExist" + ); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testSameProtocols() throws Exception { + protocols = new String[] { + "TLSv1.1", + "TLSv1.2" + }; + + startGrid(); + + checkSuccessfulClientStart( + null, + new String[] { + "TLSv1.1", + "TLSv1.2" + } + ); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testOneCommonProtocol() throws Exception { + protocols = new String[] { + "TLSv1", + "TLSv1.1", + "TLSv1.2" + }; + + startGrid(); + + checkSuccessfulClientStart( + null, + new String[] { + "TLSv1.1", + "SSLv3" + } + ); + } + + /** + * @param cipherSuites list of cipher suites + * @param protocols list of protocols + * @throws Exception If failed. + */ + private void checkSuccessfulClientStart(String[] cipherSuites, String[] protocols) throws Exception { + this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites; + this.protocols = F.isEmpty(protocols) ? null : protocols; + + try (GridClient client = GridClientFactory.start(getClientConfiguration())) { + List top = client.compute().refreshTopology(false, false); + + assertEquals(1, top.size()); + } + } + + /** + * @param cipherSuites list of cipher suites + * @param protocols list of protocols + */ + private void checkClientStartFailure(String[] cipherSuites, String[] protocols) { + checkClientStartFailure(cipherSuites, protocols, "Latest topology update failed."); + } + + /** + * @param cipherSuites list of cipher suites + * @param protocols list of protocols + * @param msg exception message + */ + private void checkClientStartFailure(String[] cipherSuites, String[] protocols, String msg) { + this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites; + this.protocols = F.isEmpty(protocols) ? null : protocols; + + GridTestUtils.assertThrows( + null, + new Callable() { + @Override public Object call() throws Exception { + GridClient client = GridClientFactory.start(getClientConfiguration()); + + client.compute().refreshTopology(false, false); + + return null; + } + }, + GridClientException.class, + msg + ); + } + +} diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java index 657fda4a80179..ffd7c81a39034 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java @@ -21,6 +21,7 @@ import org.apache.ignite.internal.TaskEventSubjectIdSelfTest; import org.apache.ignite.internal.client.ClientDefaultCacheSelfTest; import org.apache.ignite.internal.client.ClientReconnectionSelfTest; +import org.apache.ignite.internal.client.ClientSslParametersTest; import org.apache.ignite.internal.client.ClientTcpMultiThreadedSelfTest; import org.apache.ignite.internal.client.ClientTcpSslAuthenticationSelfTest; import org.apache.ignite.internal.client.ClientTcpSslMultiThreadedSelfTest; @@ -165,6 +166,9 @@ public static TestSuite suite() { suite.addTestSuite(ClientTcpTaskExecutionAfterTopologyRestartSelfTest.class); + // SSL params. + suite.addTestSuite(ClientSslParametersTest.class); + return suite; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java index 8bf1e8d58e099..27a32df246a55 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java @@ -23,17 +23,20 @@ import java.io.InputStream; import java.security.GeneralSecurityException; import java.security.KeyStore; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.Arrays; +import java.util.Collection; import javax.cache.configuration.Factory; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; +import javax.net.ssl.SSLParameters; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.ssl.SSLContextWrapper; /** * Basic ssl context factory that provides ssl context configuration with specified key @@ -87,6 +90,12 @@ public class GridSslBasicContextFactory implements GridSslContextFactory { /** Trust managers. */ private TrustManager[] trustMgrs; + /** Enabled cipher suites. */ + private String[] cipherSuites; + + /** Enabled protocols. */ + private String[] protocols; + /** * Gets key store type used for context creation. * @@ -269,6 +278,63 @@ public void setTrustManagers(TrustManager... trustMgrs) { this.trustMgrs = trustMgrs; } + /** + * Gets enabled cipher suites. + * + * @return Enabled cipher suites. + */ + public String[] getCipherSuites() { + return cipherSuites; + } + + /** + * Sets enabled cipher suites. + * + * @param cipherSuites Enabled cipher suites. + */ + public void setCipherSuites(String... cipherSuites) { + this.cipherSuites = cipherSuites; + } + + + /** + * Sets enabled cipher suites. + * + * @param cipherSuites Enabled cipher suites. + */ + public void setCipherSuites(Collection cipherSuites) { + if (!F.isEmpty(cipherSuites)) + setCipherSuites(cipherSuites.toArray(new String[0])); + } + + /** + * Gets enabled protocols. + * + * @return Enabled protocols. + */ + public String[] getProtocols() { + return protocols; + } + + /** + * Sets enabled protocols. + * + * @param protocols Enabled protocols. + */ + public void setProtocols(String... protocols) { + this.protocols = protocols; + } + + /** + * Sets enabled protocols. + * + * @param protocols Enabled protocols. + */ + public void setProtocols(Collection protocols) { + if (!F.isEmpty(protocols)) + setProtocols(protocols.toArray(new String[0])); + } + /** * Returns an instance of trust manager that will always succeed regardless of certificate provided. * @@ -303,6 +369,18 @@ public static TrustManager getDisabledTrustManager() { SSLContext ctx = SSLContext.getInstance(proto); + if (cipherSuites != null || protocols != null) { + SSLParameters sslParameters = new SSLParameters(); + + if (cipherSuites != null) + sslParameters.setCipherSuites(cipherSuites); + + if (protocols != null) + sslParameters.setProtocols(protocols); + + ctx = new SSLContextWrapper(ctx, sslParameters); + } + ctx.init(keyMgrFactory.getKeyManagers(), mgrs, null); return ctx; @@ -431,14 +509,12 @@ private static class DisabledX509TrustManager implements X509TrustManager { private static final X509Certificate[] CERTS = new X509Certificate[0]; /** {@inheritDoc} */ - @Override public void checkClientTrusted(X509Certificate[] x509Certificates, String s) - throws CertificateException { + @Override public void checkClientTrusted(X509Certificate[] x509Certificates, String s) { // No-op, all clients are trusted. } /** {@inheritDoc} */ - @Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s) - throws CertificateException { + @Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s) { // No-op, all servers are trusted. } @@ -447,4 +523,4 @@ private static class DisabledX509TrustManager implements X509TrustManager { return CERTS; } } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java index 07e7b104eed0d..abd27bdb1b315 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java @@ -71,15 +71,18 @@ public class Arguments { */ private String walArgs; - /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}. */ + /** Ping timeout for grid client. See {@link GridClientConfiguration#getPingTimeout()}. */ private long pingTimeout; - /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */ + /** Ping interval for grid client. See {@link GridClientConfiguration#getPingInterval()}. */ private long pingInterval; /** SSL Protocol. */ private String sslProtocol; + /** SSL Cipher suites. */ + private String sslCipherSuites; + /** SSL Key Algorithm. */ private String sslKeyAlgorithm; @@ -113,10 +116,11 @@ public class Arguments { * @param cacheArgs --cache subcommand arguments. * @param walAct WAL action. * @param walArgs WAL args. - * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}. - * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}. + * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#getPingTimeout()}. + * @param pingInterval Ping interval. See {@link GridClientConfiguration#getPingInterval()}. * @param autoConfirmation Auto confirmation flag. * @param sslProtocol SSL Protocol. + * @param sslCipherSuites SSL cipher suites. * @param sslKeyAlgorithm SSL Key Algorithm. * @param sslKeyStorePath Keystore. * @param sslKeyStorePassword Keystore Password. @@ -127,28 +131,39 @@ public class Arguments { */ public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct, String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs, - Long pingTimeout, Long pingInterval, boolean autoConfirmation, String sslProtocol, String sslKeyAlgorithm, + Long pingTimeout, Long pingInterval, boolean autoConfirmation, + String sslProtocol, String sslCipherSuites, String sslKeyAlgorithm, String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType, - String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType) { + String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType + ) { this.cmd = cmd; this.host = host; this.port = port; this.user = user; this.pwd = pwd; + this.baselineAct = baselineAct; this.baselineArgs = baselineArgs; + this.txArg = txArg; this.cacheArgs = cacheArgs; + this.walAct = walAct; this.walArgs = walArgs; + this.pingTimeout = pingTimeout; this.pingInterval = pingInterval; + this.autoConfirmation = autoConfirmation; + this.sslProtocol = sslProtocol; + this.sslCipherSuites = sslCipherSuites; + this.sslKeyAlgorithm = sslKeyAlgorithm; this.sslKeyStorePath = sslKeyStorePath; this.sslKeyStoreType = sslKeyStoreType; this.sslKeyStorePassword = sslKeyStorePassword; + this.sslTrustStorePath = sslTrustStorePath; this.sslTrustStoreType = sslTrustStoreType; this.sslTrustStorePassword = sslTrustStorePassword; @@ -246,7 +261,7 @@ public String walArguments() { } /** - * See {@link GridClientConfiguration#pingTimeout}. + * See {@link GridClientConfiguration#getPingInterval()}. * * @return Ping timeout. */ @@ -255,7 +270,7 @@ public long pingTimeout() { } /** - * See {@link GridClientConfiguration#pingInterval}. + * See {@link GridClientConfiguration#getPingInterval()}. * * @return Ping interval. */ @@ -277,6 +292,13 @@ public String sslProtocol() { return sslProtocol; } + /** + * @return SSL cipher suites. + */ + public String getSslCipherSuites() { + return sslCipherSuites; + } + /** * @return SSL Key Algorithm */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 1bc7f3a57400c..b363c3f745f5f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -203,6 +203,9 @@ public class CommandHandler { /** */ private static final String CMD_SSL_PROTOCOL = "--ssl-protocol"; + /** */ + private static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites"; + /** */ private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm"; @@ -229,21 +232,29 @@ public class CommandHandler { static { AUX_COMMANDS.add(CMD_HELP); + AUX_COMMANDS.add(CMD_HOST); AUX_COMMANDS.add(CMD_PORT); + AUX_COMMANDS.add(CMD_PASSWORD); AUX_COMMANDS.add(CMD_USER); + AUX_COMMANDS.add(CMD_AUTO_CONFIRMATION); + AUX_COMMANDS.add(CMD_PING_INTERVAL); AUX_COMMANDS.add(CMD_PING_TIMEOUT); + AUX_COMMANDS.add(CMD_SSL_PROTOCOL); + AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES); + AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM); + AUX_COMMANDS.add(CMD_KEYSTORE_TYPE); AUX_COMMANDS.add(CMD_KEYSTORE); AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD); - AUX_COMMANDS.add(CMD_KEYSTORE_TYPE); + + AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE); AUX_COMMANDS.add(CMD_TRUSTSTORE); AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD); - AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE); } /** Broadcast uuid. */ @@ -352,13 +363,14 @@ public class CommandHandler { private static final String UTILITY_NAME = "control.sh"; /** Common options. */ - private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), - op(CMD_PORT, "PORT"), op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), + private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"), + op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"), - op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL"), op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"), - op(CMD_KEYSTORE, "KEYSTORE"), op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"), - op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"), op(CMD_TRUSTSTORE, "TRUSTSTORE"), - op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"), op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD")); + op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"), + op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"), + op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"), + op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"), op(CMD_KEYSTORE, "KEYSTORE"), op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"), + op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"), op(CMD_TRUSTSTORE, "TRUSTSTORE"), op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD")); /** Utility name with common options. */ private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS); @@ -1033,10 +1045,9 @@ private void cachesConfig(GridClient client, CacheArguments cacheArgs, executeTaskByNameOnNode(client, VisorCacheConfigurationCollectorTask.class.getName(), taskArg, nodeId); Map cacheToMapped = - viewRes.cacheInfos().stream().collect(Collectors.toMap(x -> x.getCacheName(), x -> x.getMapped())); + viewRes.cacheInfos().stream().collect(Collectors.toMap(CacheInfo::getCacheName, CacheInfo::getMapped)); printCachesConfig(res, cacheArgs.outputFormat(), cacheToMapped); - } /** @@ -1821,18 +1832,20 @@ Arguments parseAndValidate(List rawArgs) { String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL; - String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM; + String sslCipherSuites = ""; - String sslKeyStorePath = null; + String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM; String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE; - char sslKeyStorePassword[] = null; + String sslKeyStorePath = null; - String sslTrustStorePath = null; + char sslKeyStorePassword[] = null; String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE; + String sslTrustStorePath = null; + char sslTrustStorePassword[] = null; while (hasNextArg()) { @@ -1952,38 +1965,43 @@ Arguments parseAndValidate(List rawArgs) { break; + case CMD_SSL_CIPHER_SUITES: + sslCipherSuites = nextArg("Expected SSL cipher suites"); + + break; + case CMD_SSL_KEY_ALGORITHM: sslKeyAlgorithm = nextArg("Expected SSL key algorithm"); break; case CMD_KEYSTORE: - sslKeyStorePath = nextArg("Expected keystore path"); + sslKeyStorePath = nextArg("Expected SSL key store path"); break; case CMD_KEYSTORE_PASSWORD: - sslKeyStorePassword = nextArg("Expected keystore password").toCharArray(); + sslKeyStorePassword = nextArg("Expected SSL key store password").toCharArray(); break; case CMD_KEYSTORE_TYPE: - sslKeyStoreType = nextArg("Expected keystore type"); + sslKeyStoreType = nextArg("Expected SSL key store type"); break; case CMD_TRUSTSTORE: - sslTrustStorePath = nextArg("Expected truststore path"); + sslTrustStorePath = nextArg("Expected SSL trust store path"); break; case CMD_TRUSTSTORE_PASSWORD: - sslTrustStorePassword = nextArg("Expected truststore password").toCharArray(); + sslTrustStorePassword = nextArg("Expected SSL trust store password").toCharArray(); break; case CMD_TRUSTSTORE_TYPE: - sslTrustStoreType = nextArg("Expected truststore type"); + sslTrustStoreType = nextArg("Expected SSL trust store type"); break; @@ -2008,9 +2026,13 @@ Arguments parseAndValidate(List rawArgs) { Command cmd = commands.get(0); - return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs, + return new Arguments(cmd, host, port, user, pwd, + baselineAct, baselineArgs, + txArgs, cacheArgs, + walAct, walArgs, pingTimeout, pingInterval, autoConfirmation, - sslProtocol, sslKeyAlgorithm, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType, + sslProtocol, sslCipherSuites, + sslKeyAlgorithm, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType, sslTrustStorePath, sslTrustStorePassword, sslTrustStoreType); } @@ -2492,6 +2514,23 @@ private Map mapToPairs(VisorCacheConfiguration cfg) { return params; } + /** + * Split string into items. + * + * @param s String to process. + * @param delim Delimiter. + * @return List with items. + */ + private List split(String s, String delim) { + if (F.isEmpty(s)) + return Collections.emptyList(); + + return Arrays.stream(s.split(delim)) + .map(String::trim) + .filter(item -> !item.isEmpty()) + .collect(Collectors.toList()); + } + /** * Parse and execute command. * @@ -2538,8 +2577,8 @@ public int execute(List rawArgs) { log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2)); log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2)); log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2)); - log(i("KEYSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); - log(i("TRUSTSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + log(i("KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + log(i("TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); nl(); log("Exit codes:"); @@ -2576,7 +2615,7 @@ public int execute(List rawArgs) { boolean tryConnectAgain = true; - int tryConnectMaxCount=3; + int tryConnectMaxCount = 3; while (tryConnectAgain) { tryConnectAgain = false; @@ -2598,10 +2637,21 @@ public int execute(List rawArgs) { if (!F.isEmpty(args.sslKeyStorePath())) { GridSslBasicContextFactory factory = new GridSslBasicContextFactory(); - factory.setProtocol(args.sslProtocol()); + List sslProtocols = split(args.sslProtocol(), ","); + String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0); + + factory.setProtocol(sslProtocol); factory.setKeyAlgorithm(args.sslKeyAlgorithm()); + if (sslProtocols.size() > 1) + factory.setProtocols(sslProtocols); + + factory.setCipherSuites(split(args.getSslCipherSuites(), ",")); + + if (args.sslKeyStorePath() == null) + throw new IllegalArgumentException("SSL key store location is not specified."); + factory.setKeyStoreFilePath(args.sslKeyStorePath()); if (args.sslKeyStorePassword() != null) @@ -2663,7 +2713,7 @@ public int execute(List rawArgs) { } catch (Throwable e) { if (tryConnectMaxCount > 0 && isAuthError(e)) { - System.out.println("Authentication error, try connection again."); + log("Authentication error, try connection again."); final Console console = System.console(); @@ -2676,13 +2726,13 @@ public int execute(List rawArgs) { else { Scanner scanner = new Scanner(System.in); - if (F.isEmpty(args.getUserName())){ - System.out.println("user: "); + if (F.isEmpty(args.getUserName())) { + log("user: "); args.setUserName(scanner.next()); } - System.out.println("password: "); + log("password: "); args.setPassword(scanner.next()); } diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java b/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java index 901d42b1f4ba9..67bc834d0886c 100644 --- a/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java +++ b/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java @@ -20,10 +20,15 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; -/** */ -class SSLContextWrapper extends SSLContext { - /** */ - SSLContextWrapper(SSLContext delegate, SSLParameters sslParameters) { +/** + * Wrapper for {@link SSLContext} that extend source context with custom SSL parameters. + */ +public class SSLContextWrapper extends SSLContext { + /** + * @param delegate Wrapped SSL context. + * @param sslParameters Extended SSL parameters. + */ + public SSLContextWrapper(SSLContext delegate, SSLParameters sslParameters) { super(new DelegatingSSLContextSpi(delegate, sslParameters), delegate.getProvider(), delegate.getProtocol()); diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java b/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java index edff5c9990373..fb370a57babce 100644 --- a/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java @@ -93,7 +93,7 @@ public class SslContextFactory implements Factory { /** Enabled cipher suites. */ private String[] cipherSuites; - /** Enabled cipher suites. */ + /** Enabled protocols. */ private String[] protocols; /** @@ -289,6 +289,7 @@ public static TrustManager getDisabledTrustManager() { /** * Sets enabled cipher suites. + * * @param cipherSuites enabled cipher suites. */ public void setCipherSuites(String... cipherSuites) { @@ -296,7 +297,8 @@ public void setCipherSuites(String... cipherSuites) { } /** - * Gets enabled cipher suites + * Gets enabled cipher suites. + * * @return enabled cipher suites */ public String[] getCipherSuites() { @@ -304,8 +306,9 @@ public String[] getCipherSuites() { } /** - * Gets enabled cipher suites - * @return enabled cipher suites + * Gets enabled protocols. + * + * @return Enabled protocols. */ public String[] getProtocols() { return protocols; @@ -313,7 +316,8 @@ public String[] getProtocols() { /** * Sets enabled protocols. - * @param protocols enabled protocols. + * + * @param protocols Enabled protocols. */ public void setProtocols(String... protocols) { this.protocols = protocols; @@ -515,4 +519,4 @@ private static class DisabledX509TrustManager implements X509TrustManager { throw new IgniteException(e); } } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java index 7ac6108197a24..a950d6dc10126 100644 --- a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java +++ b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java @@ -18,23 +18,28 @@ package org.apache.ignite.client; import java.util.concurrent.Callable; -import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.ClientConfiguration; import org.apache.ignite.configuration.ClientConnectorConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.ssl.SslContextFactory; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; /** * Tests cases when node connects to cluster with different set of cipher suites. */ +@RunWith(JUnit4.class) public class SslParametersTest extends GridCommonAbstractTest { - + /** */ public static final String TEST_CACHE_NAME = "TEST"; + /** */ private volatile String[] cipherSuites; @@ -58,8 +63,10 @@ public class SslParametersTest extends GridCommonAbstractTest { return cfg; } - /** {@inheritDoc} */ - protected ClientConfiguration getClientConfiguration() throws Exception { + /** + * @return Client config. + */ + protected ClientConfiguration getClientConfiguration() { ClientConfiguration cfg = new ClientConfiguration(); cfg.setAddresses("127.0.0.1:10800"); @@ -71,9 +78,11 @@ protected ClientConfiguration getClientConfiguration() throws Exception { return cfg; } + /** + * @return SSL factory. + */ @NotNull private SslContextFactory createSslFactory() { - SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory( - "node01", "trustone"); + SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory("node01", "trustone"); factory.setCipherSuites(cipherSuites); factory.setProtocols(protocols); @@ -92,6 +101,7 @@ protected ClientConfiguration getClientConfiguration() throws Exception { /** * @throws Exception If failed. */ + @Test public void testSameCipherSuite() throws Exception { cipherSuites = new String[] { "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", @@ -102,12 +112,10 @@ public void testSameCipherSuite() throws Exception { startGrid(); checkSuccessfulClientStart( - new String[][] { - new String[] { - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_RSA_WITH_AES_128_GCM_SHA256", - "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" - } + new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" }, null ); @@ -116,6 +124,7 @@ public void testSameCipherSuite() throws Exception { /** * @throws Exception If failed. */ + @Test public void testOneCommonCipherSuite() throws Exception { cipherSuites = new String[] { "TLS_RSA_WITH_AES_128_GCM_SHA256", @@ -123,13 +132,11 @@ public void testOneCommonCipherSuite() throws Exception { }; startGrid(); - + checkSuccessfulClientStart( - new String[][] { - new String[] { - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" - } + new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" }, null ); @@ -138,19 +145,18 @@ public void testOneCommonCipherSuite() throws Exception { /** * @throws Exception If failed. */ + @Test public void testNoCommonCipherSuite() throws Exception { cipherSuites = new String[] { "TLS_RSA_WITH_AES_128_GCM_SHA256" }; startGrid(); - + checkClientStartFailure( - new String[][] { - new String[] { - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" - } + new String[] { + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" }, null ); @@ -159,19 +165,18 @@ public void testNoCommonCipherSuite() throws Exception { /** * @throws Exception If failed. */ + @Test public void testNonExistentCipherSuite() throws Exception { cipherSuites = new String[] { "TLS_RSA_WITH_AES_128_GCM_SHA256" }; startGrid(); - + checkClientStartFailure( - new String[][] { - new String[] { - "TLC_FAKE_CIPHER", - "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" - } + new String[] { + "TLC_FAKE_CIPHER", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256" }, null, IllegalArgumentException.class, @@ -182,6 +187,7 @@ public void testNonExistentCipherSuite() throws Exception { /** * @throws Exception If failed. */ + @Test public void testNoCommonProtocols() throws Exception { protocols = new String[] { "TLSv1.1", @@ -192,11 +198,9 @@ public void testNoCommonProtocols() throws Exception { checkClientStartFailure( null, - new String[][] { - new String[] { - "TLSv1", - "TLSv1.2", - } + new String[] { + "TLSv1", + "TLSv1.2" } ); } @@ -204,6 +208,7 @@ public void testNoCommonProtocols() throws Exception { /** * @throws Exception If failed. */ + @Test public void testNonExistentProtocol() throws Exception { protocols = new String[] { "SSLv3" @@ -213,11 +218,9 @@ public void testNonExistentProtocol() throws Exception { checkClientStartFailure( null, - new String[][] { - new String[] { - "SSLv3", - "SSLvDoesNotExist" - } + new String[] { + "SSLv3", + "SSLvDoesNotExist" }, IllegalArgumentException.class, "SSLvDoesNotExist" @@ -227,20 +230,19 @@ public void testNonExistentProtocol() throws Exception { /** * @throws Exception If failed. */ + @Test public void testSameProtocols() throws Exception { protocols = new String[] { "TLSv1.1", - "TLSv1.2", + "TLSv1.2" }; startGrid(); checkSuccessfulClientStart(null, - new String[][] { - new String[] { - "TLSv1.1", - "TLSv1.2", - } + new String[] { + "TLSv1.1", + "TLSv1.2" } ); } @@ -248,6 +250,7 @@ public void testSameProtocols() throws Exception { /** * @throws Exception If failed. */ + @Test public void testOneCommonProtocol() throws Exception { protocols = new String[] { "TLSv1", @@ -258,11 +261,9 @@ public void testOneCommonProtocol() throws Exception { startGrid(); checkSuccessfulClientStart(null, - new String[][] { - new String[] { - "TLSv1.1", - "SSLv3" - } + new String[] { + "TLSv1.1", + "SSLv3" } ); } @@ -272,30 +273,26 @@ public void testOneCommonProtocol() throws Exception { * @param protocols list of protocols * @throws Exception If failed. */ - private void checkSuccessfulClientStart(String[][] cipherSuites, String[][] protocols) throws Exception { - int n = Math.max( - cipherSuites != null ? cipherSuites.length : 0, - protocols != null ? protocols.length : 0); - - for (int i = 0; i < n; i++) { - this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null; - this.protocols = protocols != null && i < protocols.length ? protocols[i] : null; - - IgniteClient client = Ignition.startClient(getClientConfiguration()); + private void checkSuccessfulClientStart(String[] cipherSuites, String[] protocols) throws Exception { + this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites; + this.protocols = F.isEmpty(protocols) ? null : protocols; + try (IgniteClient client = Ignition.startClient(getClientConfiguration())) { client.getOrCreateCache(TEST_CACHE_NAME); - - client.close(); } } /** * @param cipherSuites list of cipher suites * @param protocols list of protocols - * @throws Exception If failed. */ - private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols) throws Exception { - checkClientStartFailure(cipherSuites, protocols, ClientConnectionException.class, "Ignite cluster is unavailable"); + private void checkClientStartFailure(String[] cipherSuites, String[] protocols) { + checkClientStartFailure( + cipherSuites, + protocols, + ClientConnectionException.class, + "Ignite cluster is unavailable" + ); } /** @@ -303,27 +300,27 @@ private void checkClientStartFailure(String[][] cipherSuites, String[][] protoco * @param protocols list of protocols * @param ex expected exception class * @param msg exception message - * @throws Exception If failed. */ - private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols, Class ex, String msg) throws Exception { - int n = Math.max( - cipherSuites != null ? cipherSuites.length : 0, - protocols != null ? protocols.length : 0); - - for (int i = 0; i < n; i++) { - this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null; - this.protocols = protocols != null && i < protocols.length ? protocols[i] : null; - - int finalI = i; - - GridTestUtils.assertThrows(null, new Callable() { - @Override public Object call() throws Exception { + private void checkClientStartFailure( + String[] cipherSuites, + String[] protocols, + Class ex, + String msg + ) { + this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites; + this.protocols = F.isEmpty(protocols) ? null : protocols; + + GridTestUtils.assertThrows( + null, + new Callable() { + @Override public Object call() { Ignition.startClient(getClientConfiguration()); return null; } - }, ex, msg); - } + }, + ex, + msg + ); } - } diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java index 59155e5b3713a..9616d51f4b87e 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java @@ -17,28 +17,39 @@ package org.apache.ignite.util; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.apache.ignite.Ignite; import org.apache.ignite.configuration.ConnectorConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.commandline.CommandHandler; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.ssl.SslContextFactory; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.NotNull; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_CONNECTION_FAILED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; /** - * Command line handler test. + * Command line handler test with SSL. */ +@RunWith(JUnit4.class) public class GridCommandHandlerSslTest extends GridCommonAbstractTest { + /** */ + private volatile String[] cipherSuites; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { cleanPersistenceDir(); stopAllGrids(); - } /** {@inheritDoc} */ @@ -46,7 +57,17 @@ public class GridCommandHandlerSslTest extends GridCommonAbstractTest { stopAllGrids(); cleanPersistenceDir(); + } + /** + * @return SSL factory. + */ + @NotNull private SslContextFactory createSslFactory() { + SslContextFactory factory = (SslContextFactory)GridTestUtils.sslFactory(); + + factory.setCipherSuites(cipherSuites); + + return factory; } /** {@inheritDoc} */ @@ -59,30 +80,94 @@ public class GridCommandHandlerSslTest extends GridCommonAbstractTest { cfg.setConnectorConfiguration(new ConnectorConfiguration()); cfg.getConnectorConfiguration().setSslEnabled(true); - cfg.setSslContextFactory(GridTestUtils.sslFactory()); + cfg.setSslContextFactory(createSslFactory()); return cfg; } /** - * Test activation works via control.sh - * + * @param nodeCipherSuite Ciphers suites to set on node. + * @param utilityCipherSuite Ciphers suites to set on utility. + * @param expRes Expected result. * @throws Exception If failed. */ - public void testActivate() throws Exception { + private void activate(String nodeCipherSuite, String utilityCipherSuite, int expRes) throws Exception { + cipherSuites = F.isEmpty(nodeCipherSuite) ? null : nodeCipherSuite.split(","); + Ignite ignite = startGrids(1); assertFalse(ignite.cluster().active()); final CommandHandler cmd = new CommandHandler(); - assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList( - "--activate", - "--keystore", GridTestUtils.keyStorePath("node01"), - "--keystore-password", GridTestUtils.keyStorePassword()))); - assertTrue(ignite.cluster().active()); + List params = new ArrayList<>(); + params.add("--activate"); + params.add("--keystore"); + params.add(GridTestUtils.keyStorePath("node01")); + params.add("--keystore-password"); + params.add(GridTestUtils.keyStorePassword()); + + if (!F.isEmpty(utilityCipherSuite)) { + params.add("--ssl-cipher-suites"); + params.add(utilityCipherSuite); + } + + assertEquals(expRes, cmd.execute(params)); + + if (expRes == EXIT_CODE_OK) + assertTrue(ignite.cluster().active()); + else + assertFalse(ignite.cluster().active()); assertEquals(EXIT_CODE_CONNECTION_FAILED, cmd.execute(Arrays.asList("--deactivate", "--yes"))); } + /** + * @throws Exception If test failed. + */ + @Test + public void testDefaultCipherSuite() throws Exception { + cipherSuites = null; + + activate(null, null, EXIT_CODE_OK); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testSameCipherSuite() throws Exception { + String ciphers = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," + + "TLS_RSA_WITH_AES_128_GCM_SHA256," + + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"; + + activate(ciphers, ciphers, EXIT_CODE_OK); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testOneCommonCipherSuite() throws Exception { + String nodeCipherSuites = "TLS_RSA_WITH_AES_128_GCM_SHA256," + + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"; + + String utilityCipherSuites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," + + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"; + + activate(nodeCipherSuites, utilityCipherSuites, EXIT_CODE_OK); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testNoCommonCipherSuite() throws Exception { + String nodeCipherSuites = "TLS_RSA_WITH_AES_128_GCM_SHA256"; + + String utilityCipherSuites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," + + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"; + + activate(nodeCipherSuites, utilityCipherSuites, EXIT_CODE_CONNECTION_FAILED); + } } From e4a1f5278eb8f67d9e23a4516c01162ba101656e Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 12 Dec 2018 21:20:50 +0700 Subject: [PATCH 242/403] IGNITE-10189 Fixed tests. (cherry picked from commit 20ffb764180a160f8c43037466f59bedde2bfdf0) --- .../ignite/internal/client/ClientSslParametersTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java index d865e5d8d028f..9f93999eebaf8 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java @@ -181,6 +181,8 @@ public void testNoCommonCipherSuite() throws Exception { @Test @Ignore("https://issues.apache.org/jira/browse/IGNITE-10245") public void testNonExistentCipherSuite() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10245"); + cipherSuites = new String[] { "TLS_RSA_WITH_AES_128_GCM_SHA256" }; @@ -224,6 +226,8 @@ public void testNoCommonProtocols() throws Exception { @Test @Ignore("https://issues.apache.org/jira/browse/IGNITE-10245") public void testNonExistentProtocol() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10245"); + protocols = new String[] { "SSLv3" }; From d0cab40f59581877e0d5619d138bbf00c0152220 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 14 Dec 2018 10:53:13 +0700 Subject: [PATCH 243/403] IGNITE-10189 Minor fixes of code style. (cherry picked from commit 90508b89593a11cecd6d4d948d6d4bdb4c5e0eff) --- .../apache/ignite/internal/client/ClientSslParametersTest.java | 3 +-- .../org/apache/ignite/internal/commandline/CommandHandler.java | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java index 9f93999eebaf8..c2e10b8135557 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java @@ -333,7 +333,6 @@ private void checkClientStartFailure(String[] cipherSuites, String[] protocols, }, GridClientException.class, msg - ); + ); } - } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index b363c3f745f5f..8d36769c92794 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -642,7 +642,6 @@ private R executeTask(GridClient client, Class> /** * @param client Client. - * * @return List of hosts. */ private Stream> listHosts(GridClient client) throws GridClientException { @@ -656,7 +655,6 @@ private Stream> listHosts(GridClient clien /** * @param client Client. - * * @return List of hosts. */ private Stream>> listHostsByClientNode(GridClient client) throws GridClientException { From 6299f104fb9d04ed5411a85f7282df1d1da53f97 Mon Sep 17 00:00:00 2001 From: Slava Koptilin Date: Thu, 13 Dec 2018 17:45:58 +0300 Subject: [PATCH 244/403] IGNITE-10514 Cache validation on the primary node may result in AssertionError - Fixes #5558. Signed-off-by: Ivan Rakov (cherry picked from commit 1039c8a7921858c7d577816ef1994de969d18e74) --- .../dht/atomic/GridDhtAtomicCache.java | 52 +++++++++++++++++-- .../colocated/GridDhtColocatedLockFuture.java | 13 ++++- .../distributed/near/GridNearLockFuture.java | 12 ++++- .../near/GridNearTxAbstractEnlistFuture.java | 13 ++++- .../near/TxTopologyVersionFuture.java | 13 ++++- 5 files changed, 96 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 109c2c29e9c2b..a645db5306f67 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -73,6 +73,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache; @@ -1722,9 +1723,9 @@ private void onForceKeysError(final UUID nodeId, * @param completionCb Completion callback. */ private void updateAllAsyncInternal0( - ClusterNode node, - GridNearAtomicAbstractUpdateRequest req, - UpdateReplyClosure completionCb + final ClusterNode node, + final GridNearAtomicAbstractUpdateRequest req, + final UpdateReplyClosure completionCb ) { GridNearAtomicUpdateResponse res = new GridNearAtomicUpdateResponse(ctx.cacheId(), node.id(), @@ -1785,6 +1786,51 @@ private void updateAllAsyncInternal0( if (validateCache) { GridDhtTopologyFuture topFut = top.topologyVersionFuture(); + // Cache validation should use topology version from the update request + // in case of the topology version was locked on near node. + if (req.topologyLocked()) { + // affinityReadyFuture() can return GridFinishedFuture under some circumstances + // and therefore it cannot be used for validation. + IgniteInternalFuture affFut = + ctx.shared().exchange().affinityReadyFuture(req.topologyVersion()); + + if (affFut.isDone()) { + List futs = + ctx.shared().exchange().exchangeFutures(); + + boolean found = false; + + for (int i = 0; i < futs.size(); ++i) { + GridDhtPartitionsExchangeFuture fut = futs.get(i); + + // We have to check fut.exchangeDone() here - + // otherwise attempt to get topVer will throw error. + // We won't skip needed future as per affinity ready future is done. + if (fut.exchangeDone() && + fut.topologyVersion().equals(req.topologyVersion())) { + topFut = fut; + + found = true; + + break; + } + } + + assert found: "The requested topology future cannot be found [topVer=" + + req.topologyVersion() + ']'; + } + else { + affFut.listen(f -> updateAllAsyncInternal0(node, req, completionCb)); + + return; + } + + assert req.topologyVersion().equals(topFut.topologyVersion()) : + "The requested topology version cannot be found [" + + "reqTopFut=" + req.topologyVersion() + + ", topFut=" + topFut + ']'; + } + assert topFut.isDone() : topFut; Throwable err = topFut.validateCache(ctx, req.recovery(), false, null, null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java index 9dbb8be974433..95096e872e406 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java @@ -778,7 +778,18 @@ void map() { if (topVer != null) { for (GridDhtTopologyFuture fut : cctx.shared().exchange().exchangeFutures()) { if (fut.exchangeDone() && fut.topologyVersion().equals(topVer)) { - Throwable err = fut.validateCache(cctx, recovery, read, null, keys); + Throwable err = null; + + // Before cache validation, make sure that this topology future is already completed. + try { + fut.get(); + } + catch (IgniteCheckedException e) { + err = fut.error(); + } + + if (err == null) + err = fut.validateCache(cctx, recovery, read, null, keys); if (err != null) { onDone(err); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java index 6cd45148f95b1..1a7a5529a495d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java @@ -847,7 +847,17 @@ void map() { if (topVer != null) { for (GridDhtTopologyFuture fut : cctx.shared().exchange().exchangeFutures()) { if (fut.exchangeDone() && fut.topologyVersion().equals(topVer)){ - Throwable err = fut.validateCache(cctx, recovery, read, null, keys); + Throwable err = null; + + // Before cache validation, make sure that this topology future is already completed. + try { + fut.get(); + } + catch (IgniteCheckedException e) { + err = fut.error(); + } + + err = (err == null)? fut.validateCache(cctx, recovery, read, null, keys): err; if (err != null) { onDone(err); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java index bd094242a5f46..e93834b2a9a6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java @@ -223,7 +223,18 @@ else if (tx.updateLockFuture(null, this)) if (topVer != null) { for (GridDhtTopologyFuture fut : cctx.shared().exchange().exchangeFutures()) { if (fut.exchangeDone() && fut.topologyVersion().equals(topVer)) { - Throwable err = fut.validateCache(cctx, false, false, null, null); + Throwable err = null; + + // Before cache validation, make sure that this topology future is already completed. + try { + fut.get(); + } + catch (IgniteCheckedException e) { + err = fut.error(); + } + + if (err == null) + err = fut.validateCache(cctx, false, false, null, null); if (err != null) { onDone(err); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/TxTopologyVersionFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/TxTopologyVersionFuture.java index b5e38839b75b4..c13bf0ed1a4cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/TxTopologyVersionFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/TxTopologyVersionFuture.java @@ -70,7 +70,18 @@ private void init() { if (topVer != null) { for (GridDhtTopologyFuture fut : cctx.shared().exchange().exchangeFutures()) { if (fut.exchangeDone() && fut.topologyVersion().equals(topVer)) { - Throwable err = fut.validateCache(cctx, false, false, null, null); + Throwable err = null; + + // Before cache validation, make sure that this topology future is already completed. + try { + fut.get(); + } + catch (IgniteCheckedException e) { + err = fut.error(); + } + + if (err == null) + err = fut.validateCache(cctx, false, false, null, null); if (err != null) { onDone(err); From 4167966896b8efbf1f0301b64e2e13f02ca0aba9 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Fri, 14 Dec 2018 18:10:37 +0300 Subject: [PATCH 245/403] IGNITE-10680 Add the ability to use existing kernel context in standalone WAL iterator - Fixes #5669. Signed-off-by: Ivan Rakov (cherry picked from commit ab7f4a5) --- .../internal/MarshallerContextImpl.java | 8 +-- .../wal/reader/IgniteWalIteratorFactory.java | 26 ++++++++- .../reader/StandaloneWalRecordsIterator.java | 14 ++--- .../db/wal/reader/IgniteWalReaderTest.java | 56 +++++++++++++++++++ 4 files changed, 91 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java index 9bad1eacb772e..7d5bbda4100ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java @@ -576,10 +576,10 @@ public Iterator>> currentMappings() { } /** - * @return custom marshaller mapping files directory. Used for standalone WAL iteration + * @return {@code True} if marshaller context is initialized. */ - @Nullable public File getMarshallerMappingFileStoreDir() { - return marshallerMappingFileStoreDir; + public boolean initialized() { + return fileStore != null; } /** @@ -656,4 +656,4 @@ static final class CombinedMap extends AbstractMap return userMap.containsKey(key) || sysMap.containsKey(key); } } -} \ No newline at end of file +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java index 697ddeda4d052..6e5759dd88f5e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java @@ -174,7 +174,8 @@ public WALIterator iterator( iteratorParametersBuilder.validate(); return new StandaloneWalRecordsIterator(log, - prepareSharedCtx(iteratorParametersBuilder), + iteratorParametersBuilder.sharedCtx == null ? prepareSharedCtx(iteratorParametersBuilder) : + iteratorParametersBuilder.sharedCtx, iteratorParametersBuilder.ioFactory, resolveWalFiles(iteratorParametersBuilder), iteratorParametersBuilder.filter, @@ -410,6 +411,14 @@ public static class IteratorParametersBuilder { */ @Nullable private File marshallerMappingFileStoreDir; + /** + * Cache shared context. In case context is specified binary objects converting and unmarshalling will be + * performed using processors of this shared context. + *
This field can't be specified together with {@link #binaryMetadataFileStoreDir} or + * {@link #marshallerMappingFileStoreDir} fields. + * */ + @Nullable private GridCacheSharedContext sharedCtx; + /** */ @Nullable private IgniteBiPredicate filter; @@ -508,6 +517,16 @@ public IteratorParametersBuilder marshallerMappingFileStoreDir(File marshallerMa return this; } + /** + * @param sharedCtx Cache shared context. + * @return IteratorParametersBuilder Self reference. + */ + public IteratorParametersBuilder sharedContext(GridCacheSharedContext sharedCtx) { + this.sharedCtx = sharedCtx; + + return this; + } + /** * @param filter Record filter for skip records during iteration. * @return IteratorParametersBuilder Self reference. @@ -562,6 +581,7 @@ public IteratorParametersBuilder copy() { .ioFactory(ioFactory) .binaryMetadataFileStoreDir(binaryMetadataFileStoreDir) .marshallerMappingFileStoreDir(marshallerMappingFileStoreDir) + .sharedContext(sharedCtx) .from(lowBound) .to(highBound) .filter(filter) @@ -576,6 +596,10 @@ public void validate() throws IllegalArgumentException { A.ensure(U.isPow2(pageSize), "Page size must be a power of 2."); A.ensure(bufferSize >= pageSize * 2, "Buffer to small."); + + A.ensure(sharedCtx == null || (binaryMetadataFileStoreDir == null && + marshallerMappingFileStoreDir == null), "GridCacheSharedContext and binaryMetadataFileStoreDir/" + + "marshallerMappingFileStoreDir can't be specified in the same time"); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index 21275d8cd3d8d..00495e135e5a8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.reader; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -440,8 +439,7 @@ private boolean checkBounds(long idx) { final KeyCacheObject key; final CacheObject val; - final File marshallerMappingFileStoreDir = - fakeCacheObjCtx.kernalContext().marshallerContext().getMarshallerMappingFileStoreDir(); + boolean keepBinary = this.keepBinary || !fakeCacheObjCtx.kernalContext().marshallerContext().initialized(); if (dataEntry instanceof LazyDataEntry) { final LazyDataEntry lazyDataEntry = (LazyDataEntry)dataEntry; @@ -462,7 +460,7 @@ private boolean checkBounds(long idx) { val = dataEntry.value(); } - return unwrapDataEntry(fakeCacheObjCtx, dataEntry, key, val, marshallerMappingFileStoreDir); + return unwrapDataEntry(fakeCacheObjCtx, dataEntry, key, val, keepBinary); } /** @@ -471,11 +469,11 @@ private boolean checkBounds(long idx) { * @param dataEntry Data entry. * @param key Entry key. * @param val Entry value. - * @param marshallerMappingFileStoreDir Marshaller directory. + * @param keepBinary Don't convert non primitive types. * @return Unwrapped entry. */ private @NotNull DataEntry unwrapDataEntry(CacheObjectContext coCtx, DataEntry dataEntry, - KeyCacheObject key, CacheObject val, File marshallerMappingFileStoreDir) { + KeyCacheObject key, CacheObject val, boolean keepBinary) { if (dataEntry instanceof MvccDataEntry) return new UnwrapMvccDataEntry( dataEntry.cacheId(), @@ -489,7 +487,7 @@ private boolean checkBounds(long idx) { dataEntry.partitionCounter(), ((MvccDataEntry)dataEntry).mvccVer(), coCtx, - keepBinary || marshallerMappingFileStoreDir == null); + keepBinary); else return new UnwrapDataEntry( dataEntry.cacheId(), @@ -502,7 +500,7 @@ private boolean checkBounds(long idx) { dataEntry.partitionId(), dataEntry.partitionCounter(), coCtx, - keepBinary || marshallerMappingFileStoreDir == null); + keepBinary); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index beab13883a7e0..282eb3dcce132 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -38,6 +38,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -55,6 +56,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.events.WalSegmentArchivedEvent; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; @@ -809,6 +811,60 @@ public void testReadEmptyWal() throws Exception { ); } + /** + * Tests WAL iterator which uses shared cache context of currently started Ignite node. + */ + @Test + public void testIteratorWithCurrentKernelContext() throws Exception { + IgniteEx ignite = startGrid(0); + + ignite.cluster().active(true); + + int cntEntries = 100; + + putDummyRecords(ignite, cntEntries); + + String workDir = U.defaultWorkDirectory(); + + IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log); + + IteratorParametersBuilder iterParametersBuilder = + createIteratorParametersBuilder(workDir, genDbSubfolderName(ignite, 0)) + .filesOrDirs(workDir) + .binaryMetadataFileStoreDir(null) + .marshallerMappingFileStoreDir(null) + .sharedContext(ignite.context().cache().context()); + + AtomicInteger cnt = new AtomicInteger(); + + IgniteBiInClosure objConsumer = (key, val) -> { + if (val instanceof IndexedObject) { + assertEquals(key, ((IndexedObject)val).iVal); + assertEquals(key, cnt.getAndIncrement()); + } + }; + + iterateAndCountDataRecord(factory.iterator(iterParametersBuilder.copy()), objConsumer, null); + + assertEquals(cntEntries, cnt.get()); + + // Test without converting non primary types. + iterParametersBuilder.keepBinary(true); + + cnt.set(0); + + IgniteBiInClosure binObjConsumer = (key, val) -> { + if (val instanceof BinaryObject) { + assertEquals(key, ((BinaryObject)val).field("iVal")); + assertEquals(key, cnt.getAndIncrement()); + } + }; + + iterateAndCountDataRecord(factory.iterator(iterParametersBuilder.copy()), binObjConsumer, null); + + assertEquals(cntEntries, cnt.get()); + } + /** * Creates and fills cache with data. * From d05480bac3e79f18dc2ac3d57b508b332fc51ac5 Mon Sep 17 00:00:00 2001 From: pereslegin-pa Date: Thu, 18 Oct 2018 19:08:44 +0300 Subject: [PATCH 246/403] IGNITE-8570 Improved GridToStringLogger - Fixes #4786. Signed-off-by: Alexey Goncharuk (cherry picked from commit a3b624d9f34012eab1711dac14dc71af3c5bd9c4) --- .../testframework/GridStringLogger.java | 3 + .../testframework/ListeningTestLogger.java | 205 +++++++++ .../ignite/testframework/LogListener.java | 427 +++++++++++++++++ .../test/ListeningTestLoggerTest.java | 428 ++++++++++++++++++ .../testsuites/IgniteBasicTestSuite.java | 3 + 5 files changed, 1066 insertions(+) create mode 100644 modules/core/src/test/java/org/apache/ignite/testframework/ListeningTestLogger.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java index 9056dd65c1349..0b09d3e3c41ad 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java @@ -26,7 +26,10 @@ /** * Logger which logs to string buffer. + * + * @deprecated Use {@link ListeningTestLogger} instead. */ +@Deprecated public class GridStringLogger implements IgniteLogger { /** Initial string builder capacity in bytes */ private static final int INITIAL = 1024 * 33; diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/ListeningTestLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/ListeningTestLogger.java new file mode 100644 index 0000000000000..1b05f4cb642f1 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testframework/ListeningTestLogger.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testframework; + +import java.util.Collection; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.function.Consumer; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.util.typedef.X; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Implementation of {@link org.apache.ignite.IgniteLogger} that performs any actions when certain message is logged. + * It can be useful in tests to ensure that a specific message was (or was not) printed to the log. + */ +public class ListeningTestLogger implements IgniteLogger { + /** + * If set to {@code true}, enables debug and trace log messages processing. + */ + private final boolean dbg; + + /** + * Logger to echo all messages, limited by {@code dbg} flag. + */ + private final IgniteLogger echo; + + /** + * Registered log messages listeners. + */ + private final Collection> lsnrs = new CopyOnWriteArraySet<>(); + + /** + * Default constructor. + */ + public ListeningTestLogger() { + this(false); + } + + /** + * @param dbg If set to {@code true}, enables debug and trace log messages processing. + */ + public ListeningTestLogger(boolean dbg) { + this(dbg, null); + } + + /** + * @param dbg If set to {@code true}, enables debug and trace log messages processing. + * @param echo Logger to echo all messages, limited by {@code dbg} flag. + */ + public ListeningTestLogger(boolean dbg, @Nullable IgniteLogger echo) { + this.dbg = dbg; + this.echo = echo; + } + + /** + * Registers message listener. + * + * @param lsnr Message listener. + */ + public void registerListener(@NotNull LogListener lsnr) { + lsnr.reset(); + + lsnrs.add(lsnr); + } + + /** + * Registers message listener. + *

+ * NOTE listener is executed in the thread causing the logging, so it is not recommended to throw any exceptions + * from it. Use {@link LogListener} to create message predicates with assertions. + * + * @param lsnr Message listener. + * @see LogListener + */ + public void registerListener(@NotNull Consumer lsnr) { + lsnrs.add(lsnr); + } + + /** + * Unregisters message listener. + * + * @param lsnr Message listener. + */ + public void unregisterListener(@NotNull Consumer lsnr) { + lsnrs.remove(lsnr); + } + + /** + * Clears all listeners. + */ + public void clearListeners() { + lsnrs.clear(); + } + + /** {@inheritDoc} */ + @Override public ListeningTestLogger getLogger(Object ctgr) { + return this; + } + + /** {@inheritDoc} */ + @Override public void trace(String msg) { + if (!dbg) + return; + + if (echo != null) + echo.trace(msg); + + applyListeners(msg); + } + + /** {@inheritDoc} */ + @Override public void debug(String msg) { + if (!dbg) + return; + + if (echo != null) + echo.debug(msg); + + applyListeners(msg); + } + + /** {@inheritDoc} */ + @Override public void info(String msg) { + if (echo != null) + echo.info(msg); + + applyListeners(msg); + } + + /** {@inheritDoc} */ + @Override public void warning(String msg, @Nullable Throwable t) { + if (echo != null) + echo.warning(msg, t); + + applyListeners(msg); + + if (t != null) + applyListeners(X.getFullStackTrace(t)); + } + + /** {@inheritDoc} */ + @Override public void error(String msg, @Nullable Throwable t) { + if (echo != null) + echo.error(msg, t); + + applyListeners(msg); + + if (t != null) + applyListeners(X.getFullStackTrace(t)); + } + + /** {@inheritDoc} */ + @Override public boolean isTraceEnabled() { + return dbg; + } + + /** {@inheritDoc} */ + @Override public boolean isDebugEnabled() { + return dbg; + } + + /** {@inheritDoc} */ + @Override public boolean isInfoEnabled() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isQuiet() { + return false; + } + + /** {@inheritDoc} */ + @Override public String fileName() { + return null; + } + + /** + * Applies listeners whose pattern is found in the message. + * + * @param msg Message to check. + */ + private void applyListeners(String msg) { + if (msg == null) + return; + + for (Consumer lsnr : lsnrs) + lsnr.accept(msg); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java b/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java new file mode 100644 index 0000000000000..d349f066ac869 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testframework; + +import java.time.temporal.ValueRange; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * The basic listener for custom log contents checking in {@link ListeningTestLogger}.

+ * + * Supports {@link #matches(String) substring}, {@link #matches(Pattern) regular expression} or + * {@link #matches(Predicate) predicate} listeners and the following optional modifiers: + *

    + *
  • {@link Builder#times times()} sets the exact number of occurrences
  • + *
  • {@link Builder#atLeast atLeast()} sets the minimum number of occurrences
  • + *
  • {@link Builder#atMost atMost()} sets the maximum number of occurrences
  • + *
+ * {@link Builder#atLeast atLeast()} and {@link Builder#atMost atMost()} can be used together.

+ * + * If the expected number of occurrences is not specified for the listener, + * then at least one occurence is expected by default. In other words:
+ *
+ * {@code LogListener.matches(msg).build();}
+ *
+ * is equivalent to
+ *
+ * {@code LogListener.matches(msg).atLeast(1).build();}
+ * 
+ * + * If only the expected maximum number of occurrences is specified, then + * the minimum number of entries for successful validation is zero. In other words:
+ *
+ * {@code LogListener.matches(msg).atMost(10).build();}
+ *
+ * is equivalent to
+ *
+ * {@code LogListener.matches(msg).atLeast(0).atMost(10).build();}
+ * 
+ */ +public abstract class LogListener implements Consumer { + /** + * Checks that all conditions are met. + * + * @throws AssertionError If some condition failed. + */ + public abstract void check() throws AssertionError; + + /** + * Reset listener state. + */ + abstract void reset(); + + /** + * Creates new listener builder. + * + * @param substr Substring to search for in a log message. + * @return Log message listener builder. + */ + public static Builder matches(String substr) { + return new Builder().andMatches(substr); + } + + /** + * Creates new listener builder. + * + * @param regexp Regular expression to search for in a log message. + * @return Log message listener builder. + */ + public static Builder matches(Pattern regexp) { + return new Builder().andMatches(regexp); + } + + /** + * Creates new listener builder. + * + * @param pred Log message predicate. + * @return Log message listener builder. + */ + public static Builder matches(Predicate pred) { + return new Builder().andMatches(pred); + } + + /** + * Log listener builder. + */ + public static class Builder { + /** */ + private final CompositeMessageListener lsnr = new CompositeMessageListener(); + + /** */ + private Node prev; + + /** + * Add new substring predicate. + * + * @param substr Substring. + * @return current builder instance. + */ + public Builder andMatches(String substr) { + addLast(new Node(substr, msg -> { + if (substr.isEmpty()) + return msg.isEmpty() ? 1 : 0; + + int cnt = 0; + + for (int idx = 0; (idx = msg.indexOf(substr, idx)) != -1; idx++) + ++cnt; + + return cnt; + })); + + return this; + } + + /** + * Add new regular expression predicate. + * + * @param regexp Regular expression. + * @return current builder instance. + */ + public Builder andMatches(Pattern regexp) { + addLast(new Node(regexp.toString(), msg -> { + int cnt = 0; + + Matcher matcher = regexp.matcher(msg); + + while (matcher.find()) + ++cnt; + + return cnt; + })); + + return this; + } + + /** + * Add new log message predicate. + * + * @param pred Log message predicate. + * @return current builder instance. + */ + public Builder andMatches(Predicate pred) { + addLast(new Node(null, msg -> pred.test(msg) ? 1 : 0)); + + return this; + } + + /** + * Set expected number of matches.
+ * Each log message may contain several matches that will be counted, + * except {@code Predicate} which can have only one match for message. + * + * @param n Expected number of matches. + * @return current builder instance. + */ + public Builder times(int n) { + if (prev != null) + prev.cnt = n; + + return this; + } + + /** + * Set expected minimum number of matches.
+ * Each log message may contain several matches that will be counted, + * except {@code Predicate} which can have only one match for message. + * + * @param n Expected number of matches. + * @return current builder instance. + */ + public Builder atLeast(int n) { + if (prev != null) { + prev.min = n; + + prev.cnt = null; + } + + return this; + } + + /** + * Set expected maximum number of matches.
+ * Each log message may contain several matches that will be counted, + * except {@code Predicate} which can have only one match for message. + * + * @param n Expected number of matches. + * @return current builder instance. + */ + public Builder atMost(int n) { + if (prev != null) { + prev.max = n; + + prev.cnt = null; + } + + return this; + } + + /** + * Set custom message for assertion error. + * + * @param msg Custom message. + * @return current builder instance. + */ + public Builder orError(String msg) { + if (prev != null) + prev.msg = msg; + + return this; + } + + /** + * Constructs message listener. + * + * @return Log message listener. + */ + public LogListener build() { + addLast(null); + + return lsnr.lsnrs.size() == 1 ? lsnr.lsnrs.get(0) : lsnr; + } + + /** + * @param node Log listener attributes. + */ + private void addLast(Node node) { + if (prev != null) + lsnr.add(prev.listener()); + + prev = node; + } + + /** */ + private Builder() {} + + /** + * Mutable attributes for log listener. + */ + static final class Node { + /** */ + final String subj; + + /** */ + final Function func; + + /** */ + String msg; + + /** */ + Integer min; + + /** */ + Integer max; + + /** */ + Integer cnt; + + /** */ + Node(String subj, Function func) { + this.subj = subj; + this.func = func; + } + + /** */ + LogMessageListener listener() { + ValueRange range; + + if (cnt != null) + range = ValueRange.of(cnt, cnt); + else if (min == null && max == null) + range = ValueRange.of(1, Integer.MAX_VALUE); + else + range = ValueRange.of(min == null ? 0 : min, max == null ? Integer.MAX_VALUE : max); + + return new LogMessageListener(func, range, subj, msg); + } + } + } + + /** */ + private static class LogMessageListener extends LogListener { + /** */ + private final Function func; + + /** */ + private final AtomicReference err = new AtomicReference<>(); + + /** */ + private final AtomicInteger matches = new AtomicInteger(); + + /** */ + private final ValueRange exp; + + /** */ + private final String subj; + + /** */ + private final String errMsg; + + /** + * @param subj Search subject. + * @param exp Expected occurrences. + * @param func Function of counting matches in the message. + * @param errMsg Custom error message. + */ + private LogMessageListener( + @NotNull Function func, + @NotNull ValueRange exp, + @Nullable String subj, + @Nullable String errMsg + ) { + this.func = func; + this.exp = exp; + this.subj = subj == null ? func.toString() : subj; + this.errMsg = errMsg; + } + + /** {@inheritDoc} */ + @Override public void accept(String msg) { + if (err.get() != null) + return; + + try { + int cnt = func.apply(msg); + + if (cnt > 0) + matches.addAndGet(cnt); + } catch (Throwable t) { + err.compareAndSet(null, t); + + if (t instanceof VirtualMachineError) + throw t; + } + } + + /** {@inheritDoc} */ + @Override public void check() { + errCheck(); + + int matchesCnt = matches.get(); + + if (!exp.isValidIntValue(matchesCnt)) { + String err = errMsg != null ? errMsg : + "\"" + subj + "\" matches " + matchesCnt + " times, expected: " + + (exp.getMaximum() == exp.getMinimum() ? exp.getMinimum() : exp) + "."; + + throw new AssertionError(err); + } + } + + /** {@inheritDoc} */ + @Override void reset() { + matches.set(0); + } + + /** + * Check that there were no runtime errors. + */ + private void errCheck() { + Throwable t = err.get(); + + if (t instanceof Error) + throw (Error) t; + + if (t instanceof RuntimeException) + throw (RuntimeException) t; + + assert t == null : t; + } + } + + /** */ + private static class CompositeMessageListener extends LogListener { + /** */ + private final List lsnrs = new ArrayList<>(); + + /** {@inheritDoc} */ + @Override public void check() { + for (LogMessageListener lsnr : lsnrs) + lsnr.check(); + } + + /** {@inheritDoc} */ + @Override void reset() { + for (LogMessageListener lsnr : lsnrs) + lsnr.reset(); + } + + /** {@inheritDoc} */ + @Override public void accept(String msg) { + for (LogMessageListener lsnr : lsnrs) + lsnr.accept(msg); + } + + /** + * @param lsnr Listener. + */ + private void add(LogMessageListener lsnr) { + lsnrs.add(lsnr); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java new file mode 100644 index 0000000000000..a8880172d2d1c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testframework.test; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteVersionUtils; +import org.apache.ignite.logger.NullLogger; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.testframework.GridTestUtils.assertThrows; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause; + +/** + * Test. + */ +@SuppressWarnings("ThrowableNotThrown") +public class ListeningTestLoggerTest extends GridCommonAbstractTest { + /** */ + private final ListeningTestLogger log = new ListeningTestLogger(false, super.log); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setGridLogger(log); + + return cfg; + } + + /** + * Basic example of using listening logger - checks that all running instances of Ignite print product version. + * + * @throws Exception If failed. + */ + public void testIgniteVersionLogging() throws Exception { + int gridCnt = 4; + + LogListener lsnr = LogListener.matches(IgniteVersionUtils.VER_STR).atLeast(gridCnt).build(); + + log.registerListener(lsnr); + + try { + startGridsMultiThreaded(gridCnt); + + lsnr.check(); + } finally { + stopAllGrids(); + } + } + + /** + * Checks that re-register works fine. + */ + public void testUnregister() { + String msg = "catch me"; + + LogListener lsnr1 = LogListener.matches(msg).times(1).build(); + LogListener lsnr2 = LogListener.matches(msg).times(2).build(); + + log.registerListener(lsnr1); + log.registerListener(lsnr2); + + log.info(msg); + + log.unregisterListener(lsnr1); + + log.info(msg); + + lsnr1.check(); + lsnr2.check(); + + // Repeat these steps to ensure that the state is cleared during registration. + log.registerListener(lsnr1); + log.registerListener(lsnr2); + + log.info(msg); + + log.unregisterListener(lsnr1); + + log.info(msg); + + lsnr1.check(); + lsnr2.check(); + } + + /** + * Ensures that listener will be re-registered only once. + */ + public void testRegister() { + AtomicInteger cntr = new AtomicInteger(); + + LogListener lsnr3 = LogListener.matches(m -> cntr.incrementAndGet() > 0).build(); + + log.registerListener(lsnr3); + log.registerListener(lsnr3); + + log.info("1"); + + assertEquals(1, cntr.get()); + } + + /** + * Checks basic API. + */ + public void testBasicApi() { + String errMsg = "Word started with \"a\" not found."; + + LogListener lsnr = LogListener.matches(Pattern.compile("a[a-z]+")).orError(errMsg) + .andMatches("Exception message.").andMatches(".java:").build(); + + log.registerListener(lsnr); + + log.info("Something new."); + + assertThrows(log(), () -> { + lsnr.check(); + + return null; + }, AssertionError.class, errMsg); + + log.error("There was an error.", new RuntimeException("Exception message.")); + + lsnr.check(); + } + + /** + * Checks blank lines matching. + */ + public void testEmptyLine() { + LogListener emptyLineLsnr = LogListener.matches("").build(); + + log.registerListener(emptyLineLsnr); + + log.info(""); + + emptyLineLsnr.check(); + } + + /** */ + public void testPredicateExceptions() { + LogListener lsnr = LogListener.matches(msg -> { + assertFalse(msg.contains("Target")); + + return true; + }).build(); + + log.registerListener(lsnr); + + log.info("Ignored message."); + log.info("Target message."); + + assertThrowsWithCause(lsnr::check, AssertionError.class); + + // Check custom exception. + LogListener lsnr2 = LogListener.matches(msg -> { + throw new IllegalStateException("Illegal state"); + }).orError("ignored blah-blah").build(); + + log.registerListener(lsnr2); + + log.info("1"); + log.info("2"); + + assertThrowsWithCause(lsnr2::check, IllegalStateException.class); + } + + /** + * Validates listener range definition. + */ + public void testRange() { + String msg = "range"; + + LogListener lsnr2 = LogListener.matches(msg).times(2).build(); + LogListener lsnr2_3 = LogListener.matches(msg).atLeast(2).atMost(3).build(); + + log.registerListener(lsnr2); + log.registerListener(lsnr2_3); + + log.info(msg); + log.info(msg); + + lsnr2.check(); + lsnr2_3.check(); + + log.info(msg); + + assertThrowsWithCause(lsnr2::check, AssertionError.class); + + lsnr2_3.check(); + + log.info(msg); + + assertThrowsWithCause(lsnr2_3::check, AssertionError.class); + } + + /** + * Checks that substring was not found in the log messages. + */ + public void testNotPresent() { + String msg = "vacuum"; + + LogListener notPresent = LogListener.matches(msg).times(0).build(); + + log.registerListener(notPresent); + + log.info("1"); + + notPresent.check(); + + log.info(msg); + + assertThrowsWithCause(notPresent::check, AssertionError.class); + } + + /** + * Checks that the substring is found at least twice. + */ + public void testAtLeast() { + String msg = "at least"; + + LogListener atLeast2 = LogListener.matches(msg).atLeast(2).build(); + + log.registerListener(atLeast2); + + log.info(msg); + + assertThrowsWithCause(atLeast2::check, AssertionError.class); + + log.info(msg); + + atLeast2.check(); + } + + /** + * Checks that the substring is found no more than twice. + */ + public void testAtMost() { + String msg = "at most"; + + LogListener atMost2 = LogListener.matches(msg).atMost(2).build(); + + log.registerListener(atMost2); + + atMost2.check(); + + log.info(msg); + log.info(msg); + + atMost2.check(); + + log.info(msg); + + assertThrowsWithCause(atMost2::check, AssertionError.class); + } + + /** + * Checks that only last value is taken into account. + */ + public void testMultiRange() { + String msg = "multi range"; + + LogListener atMost3 = LogListener.matches(msg).times(1).times(2).atMost(3).build(); + + log.registerListener(atMost3); + + for (int i = 0; i < 6; i++) { + if (i < 4) + atMost3.check(); + else + assertThrowsWithCause(atMost3::check, AssertionError.class); + + log.info(msg); + } + + LogListener lsnr4 = LogListener.matches(msg).atLeast(2).atMost(3).times(4).build(); + + log.registerListener(lsnr4); + + for (int i = 1; i < 6; i++) { + log.info(msg); + + if (i == 4) + lsnr4.check(); + else + assertThrowsWithCause(lsnr4::check, AssertionError.class); + } + } + + /** + * Checks that matches are counted for each message. + */ + public void testMatchesPerMessage() { + LogListener lsnr = LogListener.matches("aa").times(4).build(); + + log.registerListener(lsnr); + + log.info("aabaab"); + log.info("abaaab"); + + lsnr.check(); + + LogListener newLineLsnr = LogListener.matches("\n").times(5).build(); + + log.registerListener(newLineLsnr); + + log.info("\n1\n2\n\n3\n"); + + newLineLsnr.check(); + + LogListener regexpLsnr = LogListener.matches(Pattern.compile("(?i)hi|hello")).times(3).build(); + + log.registerListener(regexpLsnr); + + log.info("Hi! Hello!"); + log.info("Hi folks"); + + regexpLsnr.check(); + } + + /** + * Check thread safety. + * + * @throws Exception If failed. + */ + public void testMultithreaded() throws Exception { + int iterCnt = 50_000; + int threadCnt = 6; + int total = threadCnt * iterCnt; + int rndNum = ThreadLocalRandom.current().nextInt(iterCnt); + + LogListener lsnr = LogListener.matches("abba").times(total) + .andMatches(Pattern.compile("(?i)abba")).times(total * 2) + .andMatches("ab").times(total) + .andMatches("ba").times(total) + .build(); + + LogListener mtLsnr = LogListener.matches("abba").build(); + + log.registerListener(lsnr); + + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < iterCnt; i++) { + if (rndNum == i) + log.registerListener(mtLsnr); + + log.info("It is the abba(ABBA) message."); + } + }, threadCnt, "test-listening-log"); + + lsnr.check(); + mtLsnr.check(); + } + + /** + * Check "echo" logger. + */ + public void testEchoLogger() { + IgniteLogger echo = new StringLogger(); + + ListeningTestLogger log = new ListeningTestLogger(true, echo); + + log.error("1"); + log.warning("2"); + log.info("3"); + log.debug("4"); + log.trace("5"); + + assertEquals("12345", echo.toString()); + } + + /** */ + private static class StringLogger extends NullLogger { + /** */ + private final StringBuilder buf = new StringBuilder(); + + /** {@inheritDoc} */ + @Override public void trace(String msg) { + buf.append(msg); + } + + /** {@inheritDoc} */ + @Override public void debug(String msg) { + buf.append(msg); + } + + /** {@inheritDoc} */ + @Override public void info(String msg) { + buf.append(msg); + } + + /** {@inheritDoc} */ + @Override public void warning(String msg, Throwable t) { + buf.append(msg); + } + + /** {@inheritDoc} */ + @Override public void error(String msg, Throwable t) { + buf.append(msg); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return buf.toString(); + } + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index 0a76ee3f794a9..40bea5a5cb3d7 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -86,6 +86,7 @@ import org.apache.ignite.startup.properties.NotStringSystemPropertyTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.test.ConfigVariationsTestSuiteBuilderTest; +import org.apache.ignite.testframework.test.ListeningTestLoggerTest; import org.apache.ignite.testframework.test.ParametersTest; import org.apache.ignite.testframework.test.VariationsIteratorTest; import org.apache.ignite.util.AttributeNodeFilterSelfTest; @@ -214,6 +215,8 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex suite.addTestSuite(CacheRebalanceConfigValidationTest.class); + suite.addTestSuite(ListeningTestLoggerTest.class); + return suite; } } From 0fc8aa006d5816631f6beb730fb7937c445602b2 Mon Sep 17 00:00:00 2001 From: NSAmelchev Date: Tue, 30 Oct 2018 18:37:51 +0300 Subject: [PATCH 247/403] IGNITE-10023 Improve ListeningTestLogger for wait conditions. - Fixes #5191. Signed-off-by: Dmitriy Pavlov (cherry picked from commit 5397d67af2c52d1acb37234c5f5247c5e5811001) --- .../ignite/testframework/LogListener.java | 76 +++++-------------- .../test/ListeningTestLoggerTest.java | 71 ++++++++--------- 2 files changed, 52 insertions(+), 95 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java b/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java index d349f066ac869..485e3fae98b35 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/LogListener.java @@ -28,7 +28,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * The basic listener for custom log contents checking in {@link ListeningTestLogger}.

@@ -66,9 +65,9 @@ public abstract class LogListener implements Consumer { /** * Checks that all conditions are met. * - * @throws AssertionError If some condition failed. + * @return {@code True} if all conditions are met. */ - public abstract void check() throws AssertionError; + public abstract boolean check(); /** * Reset listener state. @@ -122,7 +121,7 @@ public static class Builder { * @return current builder instance. */ public Builder andMatches(String substr) { - addLast(new Node(substr, msg -> { + addLast(new Node(msg -> { if (substr.isEmpty()) return msg.isEmpty() ? 1 : 0; @@ -144,7 +143,7 @@ public Builder andMatches(String substr) { * @return current builder instance. */ public Builder andMatches(Pattern regexp) { - addLast(new Node(regexp.toString(), msg -> { + addLast(new Node(msg -> { int cnt = 0; Matcher matcher = regexp.matcher(msg); @@ -165,7 +164,7 @@ public Builder andMatches(Pattern regexp) { * @return current builder instance. */ public Builder andMatches(Predicate pred) { - addLast(new Node(null, msg -> pred.test(msg) ? 1 : 0)); + addLast(new Node(msg -> pred.test(msg) ? 1 : 0)); return this; } @@ -221,19 +220,6 @@ public Builder atMost(int n) { return this; } - /** - * Set custom message for assertion error. - * - * @param msg Custom message. - * @return current builder instance. - */ - public Builder orError(String msg) { - if (prev != null) - prev.msg = msg; - - return this; - } - /** * Constructs message listener. * @@ -262,15 +248,9 @@ private Builder() {} * Mutable attributes for log listener. */ static final class Node { - /** */ - final String subj; - /** */ final Function func; - /** */ - String msg; - /** */ Integer min; @@ -281,8 +261,7 @@ static final class Node { Integer cnt; /** */ - Node(String subj, Function func) { - this.subj = subj; + Node(Function func) { this.func = func; } @@ -297,7 +276,7 @@ else if (min == null && max == null) else range = ValueRange.of(min == null ? 0 : min, max == null ? Integer.MAX_VALUE : max); - return new LogMessageListener(func, range, subj, msg); + return new LogMessageListener(func, range); } } } @@ -316,28 +295,13 @@ private static class LogMessageListener extends LogListener { /** */ private final ValueRange exp; - /** */ - private final String subj; - - /** */ - private final String errMsg; - /** - * @param subj Search subject. * @param exp Expected occurrences. * @param func Function of counting matches in the message. - * @param errMsg Custom error message. */ - private LogMessageListener( - @NotNull Function func, - @NotNull ValueRange exp, - @Nullable String subj, - @Nullable String errMsg - ) { + private LogMessageListener(@NotNull Function func, @NotNull ValueRange exp) { this.func = func; this.exp = exp; - this.subj = subj == null ? func.toString() : subj; - this.errMsg = errMsg; } /** {@inheritDoc} */ @@ -350,7 +314,8 @@ private LogMessageListener( if (cnt > 0) matches.addAndGet(cnt); - } catch (Throwable t) { + } + catch (Throwable t) { err.compareAndSet(null, t); if (t instanceof VirtualMachineError) @@ -359,18 +324,12 @@ private LogMessageListener( } /** {@inheritDoc} */ - @Override public void check() { + @Override public boolean check() { errCheck(); int matchesCnt = matches.get(); - if (!exp.isValidIntValue(matchesCnt)) { - String err = errMsg != null ? errMsg : - "\"" + subj + "\" matches " + matchesCnt + " times, expected: " + - (exp.getMaximum() == exp.getMinimum() ? exp.getMinimum() : exp) + "."; - - throw new AssertionError(err); - } + return exp.isValidIntValue(matchesCnt); } /** {@inheritDoc} */ @@ -385,10 +344,10 @@ private void errCheck() { Throwable t = err.get(); if (t instanceof Error) - throw (Error) t; + throw (Error)t; if (t instanceof RuntimeException) - throw (RuntimeException) t; + throw (RuntimeException)t; assert t == null : t; } @@ -400,9 +359,12 @@ private static class CompositeMessageListener extends LogListener { private final List lsnrs = new ArrayList<>(); /** {@inheritDoc} */ - @Override public void check() { + @Override public boolean check() { for (LogMessageListener lsnr : lsnrs) - lsnr.check(); + if (!lsnr.check()) + return false; + + return true; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java index a8880172d2d1c..8043f7e8922a3 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/test/ListeningTestLoggerTest.java @@ -29,7 +29,6 @@ import org.apache.ignite.testframework.LogListener; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import static org.apache.ignite.testframework.GridTestUtils.assertThrows; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause; /** @@ -64,7 +63,7 @@ public void testIgniteVersionLogging() throws Exception { try { startGridsMultiThreaded(gridCnt); - lsnr.check(); + assertTrue(lsnr.check()); } finally { stopAllGrids(); } @@ -88,8 +87,8 @@ public void testUnregister() { log.info(msg); - lsnr1.check(); - lsnr2.check(); + assertTrue(lsnr1.check()); + assertTrue(lsnr2.check()); // Repeat these steps to ensure that the state is cleared during registration. log.registerListener(lsnr1); @@ -101,8 +100,8 @@ public void testUnregister() { log.info(msg); - lsnr1.check(); - lsnr2.check(); + assertTrue(lsnr1.check()); + assertTrue(lsnr2.check()); } /** @@ -125,24 +124,18 @@ public void testRegister() { * Checks basic API. */ public void testBasicApi() { - String errMsg = "Word started with \"a\" not found."; - - LogListener lsnr = LogListener.matches(Pattern.compile("a[a-z]+")).orError(errMsg) + LogListener lsnr = LogListener.matches(Pattern.compile("a[a-z]+")) .andMatches("Exception message.").andMatches(".java:").build(); log.registerListener(lsnr); log.info("Something new."); - assertThrows(log(), () -> { - lsnr.check(); - - return null; - }, AssertionError.class, errMsg); + assertFalse(lsnr.check()); log.error("There was an error.", new RuntimeException("Exception message.")); - lsnr.check(); + assertTrue(lsnr.check()); } /** @@ -155,7 +148,7 @@ public void testEmptyLine() { log.info(""); - emptyLineLsnr.check(); + assertTrue(emptyLineLsnr.check()); } /** */ @@ -176,7 +169,7 @@ public void testPredicateExceptions() { // Check custom exception. LogListener lsnr2 = LogListener.matches(msg -> { throw new IllegalStateException("Illegal state"); - }).orError("ignored blah-blah").build(); + }).build(); log.registerListener(lsnr2); @@ -201,18 +194,18 @@ public void testRange() { log.info(msg); log.info(msg); - lsnr2.check(); - lsnr2_3.check(); + assertTrue(lsnr2.check()); + assertTrue(lsnr2_3.check()); log.info(msg); - assertThrowsWithCause(lsnr2::check, AssertionError.class); + assertFalse(lsnr2.check()); - lsnr2_3.check(); + assertTrue(lsnr2_3.check()); log.info(msg); - assertThrowsWithCause(lsnr2_3::check, AssertionError.class); + assertFalse(lsnr2_3.check()); } /** @@ -227,11 +220,11 @@ public void testNotPresent() { log.info("1"); - notPresent.check(); + assertTrue(notPresent.check()); log.info(msg); - assertThrowsWithCause(notPresent::check, AssertionError.class); + assertFalse(notPresent.check()); } /** @@ -246,11 +239,11 @@ public void testAtLeast() { log.info(msg); - assertThrowsWithCause(atLeast2::check, AssertionError.class); + assertFalse(atLeast2.check()); log.info(msg); - atLeast2.check(); + assertTrue(atLeast2.check()); } /** @@ -263,16 +256,16 @@ public void testAtMost() { log.registerListener(atMost2); - atMost2.check(); + assertTrue(atMost2.check()); log.info(msg); log.info(msg); - atMost2.check(); + assertTrue(atMost2.check()); log.info(msg); - assertThrowsWithCause(atMost2::check, AssertionError.class); + assertFalse(atMost2.check()); } /** @@ -287,9 +280,9 @@ public void testMultiRange() { for (int i = 0; i < 6; i++) { if (i < 4) - atMost3.check(); + assertTrue(atMost3.check()); else - assertThrowsWithCause(atMost3::check, AssertionError.class); + assertFalse(atMost3.check()); log.info(msg); } @@ -302,9 +295,9 @@ public void testMultiRange() { log.info(msg); if (i == 4) - lsnr4.check(); + assertTrue(lsnr4.check()); else - assertThrowsWithCause(lsnr4::check, AssertionError.class); + assertFalse(lsnr4.check()); } } @@ -319,7 +312,7 @@ public void testMatchesPerMessage() { log.info("aabaab"); log.info("abaaab"); - lsnr.check(); + assertTrue(lsnr.check()); LogListener newLineLsnr = LogListener.matches("\n").times(5).build(); @@ -327,7 +320,7 @@ public void testMatchesPerMessage() { log.info("\n1\n2\n\n3\n"); - newLineLsnr.check(); + assertTrue(newLineLsnr.check()); LogListener regexpLsnr = LogListener.matches(Pattern.compile("(?i)hi|hello")).times(3).build(); @@ -336,7 +329,7 @@ public void testMatchesPerMessage() { log.info("Hi! Hello!"); log.info("Hi folks"); - regexpLsnr.check(); + assertTrue(regexpLsnr.check()); } /** @@ -350,6 +343,8 @@ public void testMultithreaded() throws Exception { int total = threadCnt * iterCnt; int rndNum = ThreadLocalRandom.current().nextInt(iterCnt); + ListeningTestLogger log = new ListeningTestLogger(); + LogListener lsnr = LogListener.matches("abba").times(total) .andMatches(Pattern.compile("(?i)abba")).times(total * 2) .andMatches("ab").times(total) @@ -369,8 +364,8 @@ public void testMultithreaded() throws Exception { } }, threadCnt, "test-listening-log"); - lsnr.check(); - mtLsnr.check(); + assertTrue(lsnr.check()); + assertTrue(mtLsnr.check()); } /** From 788aa825affbb26fc1643581a4232ab56cd6f1cd Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Wed, 5 Dec 2018 13:11:17 +0300 Subject: [PATCH 248/403] IGNITE-9023 LinkageError or ClassNotFoundException should not be swollen by GridDeploymentCommunication during processing deployment request - Fixes #5472. Signed-off-by: Dmitriy Govorukhin (cherry picked from commit ee5a8f321ebef44e34fb8074944a84872d932fea) --- .../GridDeploymentCommunication.java | 10 +- ...ntRequestOfUnknownClassProcessingTest.java | 151 ++++++++++++++++++ .../testsuites/IgniteKernalSelfTestSuite.java | 2 + 3 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/managers/deployment/DeploymentRequestOfUnknownClassProcessingTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java index 2a5f7cae1224b..e14c8dfafcb10 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentCommunication.java @@ -204,9 +204,9 @@ private void processResourceRequest(UUID nodeId, GridDeploymentRequest req) { // since it was already performed before (and was successful). if (!(ldr instanceof GridDeploymentClassLoader)) { // First check for @GridNotPeerDeployable annotation. - try { - String clsName = req.resourceName().replace('/', '.'); + String clsName = req.resourceName().replace('/', '.'); + try { int idx = clsName.indexOf(".class"); if (idx >= 0) @@ -228,8 +228,10 @@ private void processResourceRequest(UUID nodeId, GridDeploymentRequest req) { return; } } - catch (ClassNotFoundException ignore) { - // Safely ignore it here - resource wasn't a class name. + catch (LinkageError | ClassNotFoundException e) { + U.warn(log, "Failed to resolve class: " + clsName, e); + // Defined errors can be safely ignored here, because of resource which is able to be not a class name. + // Unsuccessful response will be sent below if the resource failed to be loaded. } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/deployment/DeploymentRequestOfUnknownClassProcessingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/deployment/DeploymentRequestOfUnknownClassProcessingTest.java new file mode 100644 index 0000000000000..f1f080bff7e98 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/deployment/DeploymentRequestOfUnknownClassProcessingTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.managers.deployment; + +import java.net.URL; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; +import org.apache.ignite.internal.managers.communication.GridMessageListener; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestExternalClassLoader; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; +import org.apache.ignite.testframework.config.GridTestProperties; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.internal.GridTopic.TOPIC_CLASSLOAD; + +/** + * Tests the processing of deployment request with an attempt to load a class with an unknown class name. + */ +public class DeploymentRequestOfUnknownClassProcessingTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static final String TEST_TOPIC_NAME = "TEST_TOPIC_NAME"; + + /** */ + private static final String UNKNOWN_CLASS_NAME = "unknown.UnknownClassName"; + + /** */ + private final ListeningTestLogger remNodeLog = new ListeningTestLogger(false, log); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setPeerClassLoadingEnabled(true); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + startGrid(getConfiguration(getTestIgniteInstanceName(0))); + + IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(1)); + + cfg.setGridLogger(remNodeLog); + + startGrid(cfg); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** + * @throws Exception If failed. + */ + public void testResponseReceivingOnDeploymentRequestOfUnknownClass() throws Exception { + IgniteEx locNode = grid(0); + IgniteEx remNode = grid(1); + + // Register deployment on remote node for attemt to load class on request receiving + GridTestExternalClassLoader ldr = new GridTestExternalClassLoader(new URL[] { + new URL(GridTestProperties.getProperty("p2p.uri.cls")) + }); + + Class task = ldr.loadClass("org.apache.ignite.tests.p2p.P2PTestTaskExternalPath1"); + + GridDeployment locDep = remNode.context().deploy().deploy(task, task.getClassLoader()); + + final GridFutureAdapter testResultFut = new GridFutureAdapter<>(); + + final LogListener remNodeLogLsnr = LogListener + .matches(s -> s.startsWith("Failed to resolve class: " + UNKNOWN_CLASS_NAME)).build(); + + remNodeLog.registerListener(remNodeLogLsnr); + + locNode.context().io().addMessageListener(TEST_TOPIC_NAME, new GridMessageListener() { + @Override public void onMessage(UUID nodeId, Object msg, byte plc) { + try { + assertTrue(msg instanceof GridDeploymentResponse); + + GridDeploymentResponse resp = (GridDeploymentResponse)msg; + + assertFalse("Unexpected response result, success=" + resp.success(), resp.success()); + + String errMsg = resp.errorMessage(); + + assertNotNull("Response should contain an error message.", errMsg); + + assertTrue("Response contains unexpected error message, errorMessage=" + errMsg, + errMsg.startsWith("Requested resource not found (ignoring locally): " + UNKNOWN_CLASS_NAME)); + + testResultFut.onDone(); + } + catch (Error e) { + testResultFut.onDone(e); + } + } + }); + + GridDeploymentRequest req = new GridDeploymentRequest(TEST_TOPIC_NAME, locDep.classLoaderId(), + UNKNOWN_CLASS_NAME, false); + + req.responseTopicBytes(U.marshal(locNode.context(), req.responseTopic())); + + locNode.context().io().sendToGridTopic(remNode.localNode(), TOPIC_CLASSLOAD, req, GridIoPolicy.P2P_POOL); + + // Сhecks that the expected response has been received. + testResultFut.get(5_000, TimeUnit.MILLISECONDS); + + // Checks that error has been logged on remote node. + assertTrue(remNodeLogLsnr.check()); + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java index ab2306edabc16..7e0148ddf1182 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java @@ -42,6 +42,7 @@ import org.apache.ignite.internal.LongJVMPauseDetectorTest; import org.apache.ignite.internal.managers.GridManagerStopSelfTest; import org.apache.ignite.internal.managers.communication.GridCommunicationSendMessageSelfTest; +import org.apache.ignite.internal.managers.deployment.DeploymentRequestOfUnknownClassProcessingTest; import org.apache.ignite.internal.managers.deployment.GridDeploymentManagerStopSelfTest; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAliveCacheSelfTest; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManagerAttributesSelfTest; @@ -137,6 +138,7 @@ public static TestSuite suite(Set ignoredTests) throws Exception { suite.addTestSuite(IgniteConnectionConcurrentReserveAndRemoveTest.class); suite.addTestSuite(LongJVMPauseDetectorTest.class); suite.addTestSuite(ClusterMetricsSelfTest.class); + suite.addTestSuite(DeploymentRequestOfUnknownClassProcessingTest.class); // Managed Services. suite.addTestSuite(GridServiceProcessorSingleNodeSelfTest.class); From f4f943d7daf94712964e737705798532cdcaa0ac Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 17 Dec 2018 11:59:43 +0300 Subject: [PATCH 249/403] IGNITE-10624 Update deployment id for cache context after join to topology - Fixes #5629. Signed-off-by: Pavel Kovalenko (cherry picked from commit 8ef84661f450f775f1f6b2ad7d893fead75d97aa) Signed-off-by: Pavel Kovalenko --- .../processors/cache/CacheGroupContext.java | 17 +- .../processors/cache/GridCacheContext.java | 24 +- .../processors/cache/GridCacheProcessor.java | 26 +- .../processors/query/GridQueryProcessor.java | 2 +- .../db/IgniteLogicalRecoveryTest.java | 50 +++ .../db/wal/reader/IgniteWalReaderTest.java | 1 - .../hashmap/GridCacheTestContext.java | 3 + .../cache/index/BasicIndexMultinodeTest.java | 28 ++ .../cache/index/BasicIndexTest.java | 344 ++++++++++-------- .../IgniteCacheQuerySelfTestSuite.java | 2 + 10 files changed, 309 insertions(+), 188 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexMultinodeTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index ea594bc7dc6e1..fdb30b7f2d95f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -765,19 +765,20 @@ public void finishRecovery( UUID originalReceivedFrom, boolean affinityNode ) throws IgniteCheckedException { - if (recoveryMode.compareAndSet(true, false)) { - affNode = affinityNode; + if (!recoveryMode.compareAndSet(true, false)) + return; - rcvdFrom = originalReceivedFrom; + affNode = affinityNode; - locStartVer = startVer; + rcvdFrom = originalReceivedFrom; - persistGlobalWalState(globalWalEnabled); + locStartVer = startVer; - initializeIO(); + persistGlobalWalState(globalWalEnabled); - ctx.affinity().onCacheGroupCreated(this); - } + initializeIO(); + + ctx.affinity().onCacheGroupCreated(this); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 31da6b900171a..542f8125a18f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -242,7 +242,7 @@ public class GridCacheContext implements Externalizable { private volatile AffinityTopologyVersion locStartTopVer; /** Dynamic cache deployment ID. */ - private IgniteUuid dynamicDeploymentId; + private volatile IgniteUuid dynamicDeploymentId; /** Updates allowed flag. */ private boolean updatesAllowed; @@ -313,8 +313,10 @@ public GridCacheContext( CacheGroupContext grp, CacheType cacheType, AffinityTopologyVersion locStartTopVer, + IgniteUuid deploymentId, boolean affNode, boolean updatesAllowed, + boolean statisticsEnabled, boolean recoveryMode, /* @@ -400,8 +402,11 @@ public GridCacheContext( readFromBackup = cacheCfg.isReadFromBackup(); + this.dynamicDeploymentId = deploymentId; this.recoveryMode = recoveryMode; + statisticsEnabled(statisticsEnabled); + assert kernalContext().recoveryMode() == recoveryMode; if (!recoveryMode) { @@ -415,10 +420,9 @@ public GridCacheContext( * Called when cache was restored during recovery and node has joined to topology. * * @param topVer Cache topology join version. - * @param statisticsEnabled Flag indicates is statistics enabled or not for that cache. - * Value may be changed after node joined to topology. + * @param clusterWideDesc Cluster-wide cache descriptor received during exchange. */ - public void finishRecovery(AffinityTopologyVersion topVer, boolean statisticsEnabled) { + public void finishRecovery(AffinityTopologyVersion topVer, DynamicCacheDescriptor clusterWideDesc) { assert recoveryMode : this; recoveryMode = false; @@ -427,9 +431,10 @@ public void finishRecovery(AffinityTopologyVersion topVer, boolean statisticsEna locMacs = localNode().attribute(ATTR_MACS); - this.statisticsEnabled = statisticsEnabled; - assert locMacs != null; + + this.statisticsEnabled = clusterWideDesc.cacheConfiguration().isStatisticsEnabled(); + this.dynamicDeploymentId = clusterWideDesc.deploymentId(); } /** @@ -460,13 +465,6 @@ public boolean customAffinityMapper() { return customAffMapper; } - /** - * @param dynamicDeploymentId Dynamic deployment ID. - */ - void dynamicDeploymentId(IgniteUuid dynamicDeploymentId) { - this.dynamicDeploymentId = dynamicDeploymentId; - } - /** * @return Dynamic deployment ID. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index ef4938157d247..91f3f17dfe48e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -1565,8 +1565,10 @@ private void onKernalStop(GridCacheAdapter cache, boolean cancel) { grp, desc.cacheType(), locStartTopVer, + desc.deploymentId(), affNode, updatesAllowed, + desc.cacheConfiguration().isStatisticsEnabled(), recoveryMode, /* * Managers in starting order! @@ -1585,8 +1587,6 @@ private void onKernalStop(GridCacheAdapter cache, boolean cancel) { affMgr ); - cacheCtx.statisticsEnabled(desc.cacheConfiguration().isStatisticsEnabled()); - cacheCtx.cacheObjectContext(cacheObjCtx); GridCacheAdapter cache = null; @@ -1701,8 +1701,10 @@ private void onKernalStop(GridCacheAdapter cache, boolean cancel) { grp, desc.cacheType(), locStartTopVer, + desc.deploymentId(), affNode, true, + desc.cacheConfiguration().isStatisticsEnabled(), recoveryMode, /* * Managers in starting order! @@ -1721,8 +1723,6 @@ private void onKernalStop(GridCacheAdapter cache, boolean cancel) { affMgr ); - cacheCtx.statisticsEnabled(desc.cacheConfiguration().isStatisticsEnabled()); - cacheCtx.cacheObjectContext(cacheObjCtx); GridDhtCacheAdapter dht = null; @@ -2211,7 +2211,7 @@ public void prepareCacheStart( if (cacheCtx.isRecoveryMode()) finishRecovery(exchTopVer, cacheCtx); else { - ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); + ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema()); onCacheStarted(cacheCtx); } @@ -2276,7 +2276,7 @@ private GridCacheContext prepareCacheContext( false ); - initCacheContext(cacheCtx, ccfg, desc.deploymentId()); + initCacheContext(cacheCtx, ccfg); return cacheCtx; } @@ -2307,7 +2307,9 @@ private void stopCacheSafely(GridCacheContext cctx) { * @param cacheContext Cache context. * @throws IgniteCheckedException If failed. */ - private void finishRecovery(AffinityTopologyVersion cacheStartVer, GridCacheContext cacheContext) throws IgniteCheckedException { + private void finishRecovery( + AffinityTopologyVersion cacheStartVer, GridCacheContext cacheContext + ) throws IgniteCheckedException { CacheGroupContext groupContext = cacheContext.group(); // Take cluster-wide cache descriptor and try to update local cache and cache group parameters. @@ -2319,7 +2321,7 @@ private void finishRecovery(AffinityTopologyVersion cacheStartVer, GridCacheCont isLocalAffinity(updatedDescriptor.cacheConfiguration()) ); - cacheContext.finishRecovery(cacheStartVer, updatedDescriptor.cacheConfiguration().isStatisticsEnabled()); + cacheContext.finishRecovery(cacheStartVer, updatedDescriptor); onKernalStart(cacheContext.cache()); @@ -2439,16 +2441,12 @@ private CacheGroupContext getOrCreateCacheGroupContext( * * @param cacheCtx Cache context to initializtion. * @param cfg Cache configuration. - * @param deploymentId Dynamic deployment ID. * @throws IgniteCheckedException if failed. */ private void initCacheContext( GridCacheContext cacheCtx, - CacheConfiguration cfg, - IgniteUuid deploymentId + CacheConfiguration cfg ) throws IgniteCheckedException { - cacheCtx.dynamicDeploymentId(deploymentId); - GridCacheAdapter cache = cacheCtx.cache(); sharedCtx.addCacheContext(cacheCtx); @@ -2556,7 +2554,7 @@ private void onCacheStarted(GridCacheContext cacheCtx) throws IgniteCheckedExcep true ); - initCacheContext(cacheCtx, cfg, desc.deploymentId()); + initCacheContext(cacheCtx, cfg); cacheCtx.onStarted(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 4ed53b2e9c5d9..7e1cc48636df2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -149,7 +149,7 @@ public class GridQueryProcessor extends GridProcessorAdapter { private final ConcurrentMap typesByName = new ConcurrentHashMap<>(); /** */ - private final GridQueryIndexing idx; + private final @Nullable GridQueryIndexing idx; /** Value object context. */ private final CacheQueryObjectValueContext valCtx; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java index 530e4c4fb975d..4a4c82193e518 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java @@ -23,6 +23,7 @@ import java.nio.file.OpenOption; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -203,6 +204,8 @@ public void testRecoveryOnJoinToActiveCluster() throws Exception { cacheLoader.consistencyCheck(node); checkNoRebalanceAfterRecovery(); + + checkCacheContextsConsistencyAfterRecovery(); } /** @@ -236,6 +239,8 @@ public void testRecoveryOnJoinToInactiveCluster() throws Exception { checkNoRebalanceAfterRecovery(); cacheLoader.consistencyCheck(node); + + checkCacheContextsConsistencyAfterRecovery(); } /** @@ -296,6 +301,8 @@ private void doTestWithDynamicCaches(List dynamicCaches) thr for (int idx = 0; idx < 3; idx++) cacheLoader.consistencyCheck(grid(idx)); + + checkCacheContextsConsistencyAfterRecovery(); } /** @@ -328,6 +335,8 @@ public void testRecoveryOnJoinToDifferentBlt() throws Exception { for (int idx = 0; idx < 3; idx++) cacheLoader.consistencyCheck(grid(idx)); + + checkCacheContextsConsistencyAfterRecovery(); } /** @@ -384,6 +393,47 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { cacheLoader.consistencyCheck(grid(idx)); } + /** + * Checks that cache contexts have consistent parameters after recovery finished and nodes have joined to topology. + */ + private void checkCacheContextsConsistencyAfterRecovery() throws Exception { + IgniteEx crd = grid(0); + + Collection cacheNames = crd.cacheNames(); + + for (String cacheName : cacheNames) { + for (int nodeIdx = 1; nodeIdx < 3; nodeIdx++) { + IgniteEx node = grid(nodeIdx); + + GridCacheContext one = cacheContext(crd, cacheName); + GridCacheContext other = cacheContext(node, cacheName); + + checkCacheContextsConsistency(one, other); + } + } + } + + /** + * @return Cache context with given name from node. + */ + private GridCacheContext cacheContext(IgniteEx node, String cacheName) { + return node.cachex(cacheName).context(); + } + + /** + * Checks that cluster-wide parameters are consistent between two caches. + * + * @param one Cache context. + * @param other Cache context. + */ + private void checkCacheContextsConsistency(GridCacheContext one, GridCacheContext other) { + Assert.assertEquals(one.statisticsEnabled(), other.statisticsEnabled()); + Assert.assertEquals(one.dynamicDeploymentId(), other.dynamicDeploymentId()); + Assert.assertEquals(one.keepBinary(), other.keepBinary()); + Assert.assertEquals(one.updatesAllowed(), other.updatesAllowed()); + Assert.assertEquals(one.group().receivedFrom(), other.group().receivedFrom()); + } + /** {@inheritDoc} */ @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { return new StopNodeFailureHandler(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index 282eb3dcce132..f6f8a953820f0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -814,7 +814,6 @@ public void testReadEmptyWal() throws Exception { /** * Tests WAL iterator which uses shared cache context of currently started Ignite node. */ - @Test public void testIteratorWithCurrentKernelContext() throws Exception { IgniteEx ignite = startGrid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java index 4159f6ed742a4..55c4c4574fc5d 100644 --- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java +++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java @@ -48,6 +48,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager; import org.apache.ignite.internal.processors.plugin.CachePluginManager; +import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.testframework.junits.GridTestKernalContext; import static org.apache.ignite.testframework.junits.GridAbstractTest.defaultCacheConfiguration; @@ -88,9 +89,11 @@ public GridCacheTestContext(GridTestKernalContext ctx) throws Exception { null, CacheType.USER, AffinityTopologyVersion.ZERO, + IgniteUuid.randomUuid(), true, true, false, + false, new GridCacheEventManager(), new CacheOsStoreManager(null, new CacheConfiguration()), new GridCacheEvictionManager(), diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexMultinodeTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexMultinodeTest.java new file mode 100644 index 0000000000000..37c82c2156c84 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexMultinodeTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.index; + +/** + * A set of basic tests for caches with indexes in multi-node environment. + */ +public class BasicIndexMultinodeTest extends BasicIndexTest { + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 3; + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexTest.java index feb330bcaef87..df9e988a81a05 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/BasicIndexTest.java @@ -25,8 +25,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CacheKeyConfiguration; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.QueryIndex; import org.apache.ignite.cache.query.SqlFieldsQuery; @@ -34,8 +34,10 @@ import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -51,31 +53,30 @@ public class BasicIndexTest extends GridCommonAbstractTest { private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); /** */ - private Collection indexes; + private Collection indexes = Collections.emptyList(); /** */ private Integer inlineSize; /** */ - private Boolean isPersistenceEnabled; + private boolean isPersistenceEnabled; /** */ - private String affKeyFieldName; + private int gridCount = 1; - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - assertNotNull(indexes); - assertNotNull(inlineSize); - assertNotNull(isPersistenceEnabled); - - for (QueryIndex index : indexes) { + for (QueryIndex index : indexes) index.setInlineSize(inlineSize); - } IgniteConfiguration igniteCfg = super.getConfiguration(igniteInstanceName); + igniteCfg.setConsistentId(igniteInstanceName); + igniteCfg.setDiscoverySpi( new TcpDiscoverySpi().setIpFinder(IP_FINDER) ); @@ -98,13 +99,6 @@ public class BasicIndexTest extends GridCommonAbstractTest { )) .setSqlIndexMaxInlineSize(inlineSize); - if (affKeyFieldName != null) { - ccfg.setKeyConfiguration(new CacheKeyConfiguration() - .setTypeName(Key.class.getTypeName()) - .setAffinityKeyFieldName(affKeyFieldName) - ); - } - igniteCfg.setCacheConfiguration(ccfg); if (isPersistenceEnabled) { @@ -118,7 +112,9 @@ public class BasicIndexTest extends GridCommonAbstractTest { return igniteCfg; } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -127,43 +123,40 @@ public class BasicIndexTest extends GridCommonAbstractTest { cleanPersistenceDir(); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected void afterTest() throws Exception { stopAllGrids(); cleanPersistenceDir(); - indexes = null; - - inlineSize = null; - - isPersistenceEnabled = null; - - affKeyFieldName = null; - super.afterTest(); } + /** + * @return Grid count used in test. + */ + protected int gridCount() { + return gridCount; + } + /** */ public void testNoIndexesNoPersistence() throws Exception { - indexes = Collections.emptyList(); - - isPersistenceEnabled = false; - - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); + startGridsMultiThreaded(gridCount()); populateCache(); checkAll(); - stopGrid(); + stopAllGrids(); } } @@ -178,39 +171,33 @@ public void testAllIndexesNoPersistence() throws Exception { new QueryIndex("valPojo") ); - isPersistenceEnabled = false; - - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); + startGridsMultiThreaded(gridCount()); populateCache(); checkAll(); - stopGrid(); + stopAllGrids(); } } /** */ public void testDynamicIndexesNoPersistence() throws Exception { - indexes = Collections.emptyList(); - - isPersistenceEnabled = false; - - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); + startGridsMultiThreaded(gridCount()); populateCache(); @@ -225,40 +212,34 @@ public void testDynamicIndexesNoPersistence() throws Exception { checkAll(); - stopGrid(); + stopAllGrids(); } } /** */ public void testNoIndexesWithPersistence() throws Exception { - indexes = Collections.emptyList(); - isPersistenceEnabled = true; - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); populateCache(); checkAll(); - stopGrid(); + stopAllGrids(); - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); checkAll(); - stopGrid(); + stopAllGrids(); cleanPersistenceDir(); } @@ -277,30 +258,26 @@ public void testAllIndexesWithPersistence() throws Exception { isPersistenceEnabled = true; - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); populateCache(); checkAll(); - stopGrid(); - - startGrid(); + stopAllGrids(); - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); checkAll(); - stopGrid(); + stopAllGrids(); cleanPersistenceDir(); } @@ -308,20 +285,16 @@ public void testAllIndexesWithPersistence() throws Exception { /** */ public void testDynamicIndexesWithPersistence() throws Exception { - indexes = Collections.emptyList(); - isPersistenceEnabled = true; - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); populateCache(); @@ -336,15 +309,57 @@ public void testDynamicIndexesWithPersistence() throws Exception { checkAll(); - stopGrid(); + stopAllGrids(); - startGrid(); + startGridsMultiThreaded(gridCount()); - grid().cluster().active(true); + checkAll(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + } + + /** */ + public void testDynamicIndexesDropWithPersistence() throws Exception { + isPersistenceEnabled = true; + + int[] inlineSizes = {0, 10, 20, 50, 100}; + + for (int i : inlineSizes) { + log().info("Checking inlineSize=" + i); + + inlineSize = i; + + startGridsMultiThreaded(gridCount()); + + populateCache(); + + String[] cols = { + "keyStr", + "keyLong", + "keyPojo", + "valStr", + "valLong", + "valPojo" + }; + + createDynamicIndexes(cols); + + checkAll(); + + dropDynamicIndexes(cols); + + checkAll(); + + stopAllGrids(); + + startGridsMultiThreaded(gridCount()); checkAll(); - stopGrid(); + stopAllGrids(); cleanPersistenceDir(); } @@ -352,44 +367,38 @@ public void testDynamicIndexesWithPersistence() throws Exception { /** */ public void testNoIndexesWithPersistenceIndexRebuild() throws Exception { - indexes = Collections.emptyList(); - isPersistenceEnabled = true; - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); populateCache(); checkAll(); - Path idxPath = getIndexBinPath(); + List idxPaths = getIndexBinPaths(); // Shutdown gracefully to ensure there is a checkpoint with index.bin. // Otherwise index.bin rebuilding may not work. - grid().cluster().active(false); - - stopGrid(); + grid(0).cluster().active(false); - assertTrue(U.delete(idxPath)); + stopAllGrids(); - startGrid(); + idxPaths.forEach(idxPath -> assertTrue(U.delete(idxPath))); - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); - grid().cache(DEFAULT_CACHE_NAME).indexReadyFuture().get(); + grid(0).cache(DEFAULT_CACHE_NAME).indexReadyFuture().get(); checkAll(); - stopGrid(); + stopAllGrids(); cleanPersistenceDir(); } @@ -408,40 +417,36 @@ public void testAllIndexesWithPersistenceIndexRebuild() throws Exception { isPersistenceEnabled = true; - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); populateCache(); checkAll(); - Path idxPath = getIndexBinPath(); + List idxPaths = getIndexBinPaths(); // Shutdown gracefully to ensure there is a checkpoint with index.bin. // Otherwise index.bin rebuilding may not work. - grid().cluster().active(false); - - stopGrid(); + grid(0).cluster().active(false); - assertTrue(U.delete(idxPath)); + stopAllGrids(); - startGrid(); + idxPaths.forEach(idxPath -> assertTrue(U.delete(idxPath))); - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); - grid().cache(DEFAULT_CACHE_NAME).indexReadyFuture().get(); + grid(0).cache(DEFAULT_CACHE_NAME).indexReadyFuture().get(); checkAll(); - stopGrid(); + stopAllGrids(); cleanPersistenceDir(); } @@ -449,20 +454,16 @@ public void testAllIndexesWithPersistenceIndexRebuild() throws Exception { /** */ public void testDynamicIndexesWithPersistenceIndexRebuild() throws Exception { - indexes = Collections.emptyList(); - isPersistenceEnabled = true; - int[] inlineSizes = { 0, 10, 20, 50, 100 }; + int[] inlineSizes = {0, 10, 20, 50, 100}; for (int i : inlineSizes) { log().info("Checking inlineSize=" + i); inlineSize = i; - startGrid(); - - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); populateCache(); @@ -477,25 +478,23 @@ public void testDynamicIndexesWithPersistenceIndexRebuild() throws Exception { checkAll(); - Path idxPath = getIndexBinPath(); + List idxPaths = getIndexBinPaths(); // Shutdown gracefully to ensure there is a checkpoint with index.bin. // Otherwise index.bin rebuilding may not work. - grid().cluster().active(false); - - stopGrid(); + grid(0).cluster().active(false); - assertTrue(U.delete(idxPath)); + stopAllGrids(); - startGrid(); + idxPaths.forEach(idxPath -> assertTrue(U.delete(idxPath))); - grid().cluster().active(true); + startGridsMultiThreaded(gridCount()); - grid().cache(DEFAULT_CACHE_NAME).indexReadyFuture().get(); + grid(0).cache(DEFAULT_CACHE_NAME).indexReadyFuture().get(); checkAll(); - stopGrid(); + stopAllGrids(); cleanPersistenceDir(); } @@ -503,7 +502,7 @@ public void testDynamicIndexesWithPersistenceIndexRebuild() throws Exception { /** */ private void checkAll() { - IgniteCache cache = grid().cache(DEFAULT_CACHE_NAME); + IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); checkRemovePut(cache); @@ -520,7 +519,7 @@ private void checkAll() { /** */ private void populateCache() { - IgniteCache cache = grid().cache(DEFAULT_CACHE_NAME); + IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); // Be paranoid and populate first even indexes in ascending order, then odd indexes in descending // to check that inserting in the middle works. @@ -557,9 +556,9 @@ private void checkSelectAll(IgniteCache cache) { assertEquals(100, data.size()); for (List row : data) { - Key key = (Key)row.get(0); + Key key = (Key) row.get(0); - Val val = (Val)row.get(1); + Val val = (Val) row.get(1); long i = key.keyLong; @@ -616,9 +615,9 @@ private void checkSelectStringRange(IgniteCache cache) { assertEquals(10, data.size()); for (List row : data) { - Key key = (Key)row.get(0); + Key key = (Key) row.get(0); - Val val = (Val)row.get(1); + Val val = (Val) row.get(1); long i = key.keyLong; @@ -644,9 +643,9 @@ private void checkSelectLongRange(IgniteCache cache) { assertEquals(10, data.size()); for (List row : data) { - Key key = (Key)row.get(0); + Key key = (Key) row.get(0); - Val val = (Val)row.get(1); + Val val = (Val) row.get(1); long i = key.keyLong; @@ -658,29 +657,54 @@ private void checkSelectLongRange(IgniteCache cache) { } } - /** Must be called when the grid is up. */ - private Path getIndexBinPath() { - IgniteInternalCache cachex = grid().cachex(DEFAULT_CACHE_NAME); + /** + * Must be called when the grid is up. + */ + private List getIndexBinPaths() { + return G.allGrids().stream() + .map(grid -> (IgniteEx) grid) + .map(grid -> { + IgniteInternalCache cachex = grid.cachex(DEFAULT_CACHE_NAME); - assertNotNull(cachex); + assertNotNull(cachex); - FilePageStoreManager pageStoreMgr = (FilePageStoreManager)cachex.context().shared().pageStore(); + FilePageStoreManager pageStoreMgr = (FilePageStoreManager) cachex.context().shared().pageStore(); - assertNotNull(pageStoreMgr); + assertNotNull(pageStoreMgr); - File cacheWorkDir = pageStoreMgr.cacheWorkDir(cachex.configuration()); + File cacheWorkDir = pageStoreMgr.cacheWorkDir(cachex.configuration()); - return cacheWorkDir.toPath().resolve("index.bin"); + return cacheWorkDir.toPath().resolve("index.bin"); + }) + .collect(Collectors.toList()); } /** */ private void createDynamicIndexes(String... cols) { - IgniteCache cache = grid().cache(DEFAULT_CACHE_NAME); + IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); for (String col : cols) { + String indexName = col + "_idx"; + String schemaName = DEFAULT_CACHE_NAME; + + cache.query(new SqlFieldsQuery( + String.format("create index %s on \"%s\".Val(%s) INLINE_SIZE %s;", indexName, schemaName, col, inlineSize) + )).getAll(); + } + + cache.indexReadyFuture().get(); + } + + /** */ + private void dropDynamicIndexes(String... cols) { + IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); + + for (String col : cols) { + String indexName = col + "_idx"; + cache.query(new SqlFieldsQuery( - "create index on Val(" + col + ") INLINE_SIZE " + inlineSize - )); + String.format("drop index %s;", indexName) + )).getAll(); } cache.indexReadyFuture().get(); @@ -714,7 +738,9 @@ private Key(String str, long aLong, Pojo pojo) { keyPojo = pojo; } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public boolean equals(Object o) { if (this == o) return true; @@ -722,19 +748,23 @@ private Key(String str, long aLong, Pojo pojo) { if (o == null || getClass() != o.getClass()) return false; - Key key = (Key)o; + Key key = (Key) o; return keyLong == key.keyLong && Objects.equals(keyStr, key.keyStr) && Objects.equals(keyPojo, key.keyPojo); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public int hashCode() { return Objects.hash(keyStr, keyLong, keyPojo); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public String toString() { return S.toString(Key.class, this); } @@ -758,7 +788,9 @@ private Val(String str, long aLong, Pojo pojo) { valPojo = pojo; } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public boolean equals(Object o) { if (this == o) return true; @@ -766,19 +798,23 @@ private Val(String str, long aLong, Pojo pojo) { if (o == null || getClass() != o.getClass()) return false; - Val val = (Val)o; + Val val = (Val) o; return valLong == val.valLong && Objects.equals(valStr, val.valStr) && Objects.equals(valPojo, val.valPojo); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public int hashCode() { return Objects.hash(valStr, valLong, valPojo); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public String toString() { return S.toString(Val.class, this); } @@ -794,7 +830,9 @@ private Pojo(long pojoLong) { this.pojoLong = pojoLong; } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public boolean equals(Object o) { if (this == o) return true; @@ -802,17 +840,21 @@ private Pojo(long pojoLong) { if (o == null || getClass() != o.getClass()) return false; - Pojo pojo = (Pojo)o; + Pojo pojo = (Pojo) o; return pojoLong == pojo.pojoLong; } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public int hashCode() { return Objects.hash(pojoLong); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override public String toString() { return S.toString(Pojo.class, this); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index cd6dfc0a29c9c..6f0585bfb2853 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -106,6 +106,7 @@ import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQueryP2PDisabledSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQuerySelfTest; import org.apache.ignite.internal.processors.cache.encryption.EncryptedSqlTableTest; +import org.apache.ignite.internal.processors.cache.index.BasicIndexMultinodeTest; import org.apache.ignite.internal.processors.cache.index.BasicIndexTest; import org.apache.ignite.internal.processors.cache.index.DuplicateKeyValueClassesSelfTest; import org.apache.ignite.internal.processors.cache.index.DynamicIndexClientBasicSelfTest; @@ -234,6 +235,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(MultipleStatementsSqlQuerySelfTest.class); suite.addTestSuite(BasicIndexTest.class); + suite.addTestSuite(BasicIndexMultinodeTest.class); // Misc tests. // TODO: Enable when IGNITE-1094 is fixed. From 4fe91c36c91b7112bc8797738add0741f1baef32 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Mon, 17 Dec 2018 16:29:24 +0700 Subject: [PATCH 250/403] IGNITE-10239 Update navigation and top menu. - Fixes #5630. Signed-off-by: Andrey Novikov (cherry picked from commit 9dc251cd036a840313e3fb34aa9e522620fc2365) --- .../e2e/testcafe/components/topNavigation.js | 6 +- .../e2e/testcafe/components/userMenu.js | 2 +- .../web-console/frontend/app/app.config.js | 7 + modules/web-console/frontend/app/app.js | 8 +- .../connected-clusters-badge/style.scss | 34 ++-- .../connected-clusters-badge/template.pug | 5 +- .../icons/configuration.icon.svg | 13 ++ .../page-configure/{index.js => index.ts} | 29 +-- .../page-forgot-password/controller.js | 8 +- .../components/page-forgot-password/run.js | 11 +- .../page-forgot-password/style.scss | 38 ++-- .../page-forgot-password/template.pug | 53 +++--- .../app/components/page-landing/index.js | 11 +- .../app/components/page-landing/public.pug | 21 +++ .../app/components/page-landing/style.scss | 12 ++ .../app/components/page-landing/template.pug | 8 - .../page-password-changed/controller.ts | 32 ++++ .../components/page-password-changed/index.js | 10 +- .../page-password-changed/style.scss | 13 +- .../page-password-changed/template.pug | 9 +- .../page-password-reset/controller.js | 11 +- .../components/page-password-reset/index.js | 10 +- .../components/page-password-reset/style.scss | 15 +- .../page-password-reset/template.pug | 102 +++++------ .../page-queries/{index.js => index.ts} | 10 ++ .../app/components/page-signin/controller.ts | 7 +- .../app/components/page-signin/run.ts | 10 +- .../app/components/page-signin/style.scss | 33 ++-- .../app/components/page-signin/template.pug | 73 ++++---- .../app/components/page-signup/controller.ts | 11 +- .../app/components/page-signup/run.js | 11 +- .../app/components/page-signup/style.scss | 26 +-- .../app/components/page-signup/template.pug | 36 ++-- .../permanent-notifications/component.ts | 25 +++ .../permanent-notifications/controller.ts} | 22 +-- .../permanent-notifications/index.ts | 21 +++ .../permanent-notifications/style.scss | 54 ++++++ .../permanent-notifications/template.pug | 23 +++ .../components/timed-redirection/style.scss | 1 + .../web-console-footer/component.js | 4 +- .../style.scss => controller.ts} | 20 +-- .../components/web-console-footer/index.js | 11 +- .../components/web-console-footer/style.scss | 93 +++++++--- .../web-console-footer/template.pug | 20 ++- .../{component.js => component.ts} | 26 +-- .../component.ts} | 6 +- .../components/demo-mode-button/controller.ts | 46 +++++ .../components/demo-mode-button}/template.pug | 5 +- .../components/user-menu/component.ts} | 4 +- .../components/user-menu/controller.ts} | 56 +++--- .../components/user-menu/style.scss} | 16 +- .../components/user-menu/template.pug | 26 +++ .../web-console-header-content/component.ts | 25 +++ .../web-console-header-content/controller.ts | 45 +++++ .../web-console-header-content/style.scss} | 44 ++--- .../template.pug | 9 + .../components/web-console-header/index.js | 8 +- .../components/web-console-header/style.scss | 168 +++--------------- .../web-console-header/template.pug | 32 ++-- .../web-console-sidebar/component.ts | 25 +++ .../web-console-sidebar/controller.ts | 30 ++++ .../components/web-console-sidebar/index.ts | 25 +++ .../web-console-sidebar/style.scss} | 41 +++-- .../web-console-sidebar/template.pug | 22 +++ .../component.ts | 28 +++ .../controller.ts | 24 +++ .../web-console-sidebar-navigation/style.scss | 108 +++++++++++ .../template.pug} | 36 ++-- .../web-console-sidebar-overflow/component.ts | 26 +++ .../controller.ts | 50 ++++++ .../web-console-sidebar-overflow/style.scss | 52 ++++++ .../template.pug} | 4 +- .../app/modules/branding/branding.module.js | 10 +- .../app/modules/branding/branding.service.js | 4 - .../frontend/app/modules/demo/Demo.module.js | 28 --- .../app/modules/navbar/navbar.module.js | 9 +- .../frontend/app/services/store.ts | 75 ++++++++ .../frontend/app/store/actions/ui.ts | 36 ++++ .../actions/user.ts} | 16 +- .../effects/ui.ts} | 31 ++-- .../web-console/frontend/app/store/index.ts | 48 +++++ .../web-console/frontend/app/store/ofType.ts | 26 +++ .../frontend/app/store/reducers/ui.ts | 50 ++++++ .../frontend/app/store/reduxDebug.ts | 45 +++++ .../selectors/ui.ts} | 30 ++-- .../web-console/frontend/app/types/index.ts | 30 ++++ .../public/images/icons/cluster.icon.svg | 10 ++ .../images/icons/connectedClusters.icon.svg | 1 - .../images/icons/downloadAgent.icon.svg | 16 ++ .../frontend/public/images/icons/index.js | 5 +- .../public/images/icons/menu.icon.svg | 7 + .../frontend/public/images/icons/sql.icon.svg | 3 + .../frontend/public/stylesheets/style.scss | 74 ++++++-- modules/web-console/frontend/views/base.pug | 11 +- .../frontend/views/includes/header-left.pug | 42 ----- .../header-logo.pug => views/public.pug} | 6 +- 96 files changed, 1768 insertions(+), 811 deletions(-) create mode 100644 modules/web-console/frontend/app/components/page-configure/icons/configuration.icon.svg rename modules/web-console/frontend/app/components/page-configure/{index.js => index.ts} (91%) create mode 100644 modules/web-console/frontend/app/components/page-landing/public.pug create mode 100644 modules/web-console/frontend/app/components/page-password-changed/controller.ts rename modules/web-console/frontend/app/components/page-queries/{index.js => index.ts} (91%) create mode 100644 modules/web-console/frontend/app/components/permanent-notifications/component.ts rename modules/web-console/frontend/app/{modules/navbar/navbar.directive.js => components/permanent-notifications/controller.ts} (71%) create mode 100644 modules/web-console/frontend/app/components/permanent-notifications/index.ts create mode 100644 modules/web-console/frontend/app/components/permanent-notifications/style.scss create mode 100644 modules/web-console/frontend/app/components/permanent-notifications/template.pug rename modules/web-console/frontend/app/components/web-console-footer/{components/web-console-footer-links/style.scss => controller.ts} (70%) rename modules/web-console/frontend/app/components/web-console-header/{component.js => component.ts} (66%) rename modules/web-console/frontend/app/components/web-console-header/components/{web-console-header-extension/component.js => demo-mode-button/component.ts} (87%) create mode 100644 modules/web-console/frontend/app/components/web-console-header/components/demo-mode-button/controller.ts rename modules/web-console/frontend/app/components/{web-console-footer/components/web-console-footer-links => web-console-header/components/demo-mode-button}/template.pug (90%) rename modules/web-console/frontend/app/components/{web-console-footer/components/web-console-footer-links/component.js => web-console-header/components/user-menu/component.ts} (92%) rename modules/web-console/frontend/app/{modules/navbar/userbar.directive.js => components/web-console-header/components/user-menu/controller.ts} (50%) rename modules/web-console/frontend/app/components/{page-password-changed/controller.js => web-console-header/components/user-menu/style.scss} (81%) create mode 100644 modules/web-console/frontend/app/components/web-console-header/components/user-menu/template.pug create mode 100644 modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/component.ts create mode 100644 modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/controller.ts rename modules/web-console/frontend/app/{modules/branding/header-title.directive.js => components/web-console-header/components/web-console-header-content/style.scss} (60%) rename modules/web-console/frontend/app/components/web-console-header/components/{web-console-header-extension => web-console-header-content}/template.pug (73%) create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/component.ts create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/controller.ts create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/index.ts rename modules/web-console/frontend/app/{modules/branding/header-logo.directive.js => components/web-console-sidebar/style.scss} (57%) create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/template.pug create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/component.ts create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/controller.ts create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss rename modules/web-console/frontend/{views/includes/header-right.pug => app/components/web-console-sidebar/web-console-sidebar-navigation/template.pug} (51%) create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/component.ts create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/controller.ts create mode 100644 modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/style.scss rename modules/web-console/frontend/app/{modules/branding/powered-by-apache.pug => components/web-console-sidebar/web-console-sidebar-overflow/template.pug} (82%) create mode 100644 modules/web-console/frontend/app/services/store.ts create mode 100644 modules/web-console/frontend/app/store/actions/ui.ts rename modules/web-console/frontend/app/{modules/navbar/Navbar.provider.js => store/actions/user.ts} (80%) rename modules/web-console/frontend/app/{modules/branding/powered-by-apache.directive.js => store/effects/ui.ts} (61%) create mode 100644 modules/web-console/frontend/app/store/index.ts create mode 100644 modules/web-console/frontend/app/store/ofType.ts create mode 100644 modules/web-console/frontend/app/store/reducers/ui.ts create mode 100644 modules/web-console/frontend/app/store/reduxDebug.ts rename modules/web-console/frontend/app/{modules/branding/footer.directive.js => store/selectors/ui.ts} (62%) create mode 100644 modules/web-console/frontend/public/images/icons/cluster.icon.svg delete mode 100644 modules/web-console/frontend/public/images/icons/connectedClusters.icon.svg create mode 100644 modules/web-console/frontend/public/images/icons/downloadAgent.icon.svg create mode 100644 modules/web-console/frontend/public/images/icons/menu.icon.svg create mode 100644 modules/web-console/frontend/public/images/icons/sql.icon.svg delete mode 100644 modules/web-console/frontend/views/includes/header-left.pug rename modules/web-console/frontend/{app/modules/branding/header-logo.pug => views/public.pug} (80%) diff --git a/modules/web-console/e2e/testcafe/components/topNavigation.js b/modules/web-console/e2e/testcafe/components/topNavigation.js index b4262d0f80661..6db745e741191 100644 --- a/modules/web-console/e2e/testcafe/components/topNavigation.js +++ b/modules/web-console/e2e/testcafe/components/topNavigation.js @@ -17,5 +17,7 @@ import {Selector} from 'testcafe'; -export const configureNavButton = Selector('.wch-nav-item').withText('Configure'); -export const queriesNavButton = Selector('.wch-nav-item').withText('Queries'); +export const toggleMenuButton = Selector('.web-console-header__togle-menu-button'); + +export const configureNavButton = Selector('.web-console-sidebar-navigation__link[title="Configuration"]'); +export const queriesNavButton = Selector('.web-console-sidebar-navigation__link[title="Queries"]'); diff --git a/modules/web-console/e2e/testcafe/components/userMenu.js b/modules/web-console/e2e/testcafe/components/userMenu.js index 75246c8d6cae4..973f21f184d86 100644 --- a/modules/web-console/e2e/testcafe/components/userMenu.js +++ b/modules/web-console/e2e/testcafe/components/userMenu.js @@ -17,7 +17,7 @@ import {Selector, t} from 'testcafe'; -const _selector = Selector('[ignite-userbar]'); +const _selector = Selector('user-menu'); export const userMenu = { _selector, diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js index 88bffee7ee746..04c02ea02db20 100644 --- a/modules/web-console/frontend/app/app.config.js +++ b/modules/web-console/frontend/app/app.config.js @@ -22,6 +22,7 @@ import isNil from 'lodash/isNil'; import isEmpty from 'lodash/isEmpty'; import mixin from 'lodash/mixin'; +import {user as userAction, register as registerStore} from './store'; const nonNil = negate(isNil); const nonEmpty = negate(isEmpty); @@ -36,6 +37,8 @@ import validationTemplateUrl from 'views/templates/validation-error.tpl.pug'; const igniteConsoleCfg = angular.module('ignite-console.config', ['ngAnimate', 'mgcrea.ngStrap']); +igniteConsoleCfg.run(registerStore); + // Configure AngularJS animation: do not animate fa-spin. igniteConsoleCfg.config(['$animateProvider', ($animateProvider) => { $animateProvider.classNameFilter(/^((?!(fa-spin|ng-animate-disabled)).)*$/); @@ -133,3 +136,7 @@ igniteConsoleCfg.directive('uiGridSelection', function() { } }; }); + +igniteConsoleCfg.run(['$rootScope', 'Store', ($root, store) => { + $root.$on('user', (event, user) => store.dispatch(userAction({...user}))); +}]); diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index 2dd2733fc5689..e1cbdbb32bcf4 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -96,6 +96,7 @@ import {CSV} from './services/CSV'; import {$exceptionHandler} from './services/exceptionHandler'; import IGFSs from './services/IGFSs'; import Models from './services/Models'; +import {Store} from './services/store'; import AngularStrapTooltip from './services/AngularStrapTooltip.decorator'; import AngularStrapSelect from './services/AngularStrapSelect.decorator'; @@ -159,6 +160,8 @@ import pageSignup from './components/page-signup'; import pageSignin from './components/page-signin'; import pageForgotPassword from './components/page-forgot-password'; import formSignup from './components/form-signup'; +import sidebar from './components/web-console-sidebar'; +import permanentNotifications from './components/permanent-notifications'; import igniteServices from './services'; @@ -266,7 +269,9 @@ export default angular.module('ignite-console', [ progressLine.name, formField.name, formSignup.name, - timedRedirection.name + timedRedirection.name, + sidebar.name, + permanentNotifications.name ]) .service('$exceptionHandler', $exceptionHandler) // Directives. @@ -314,6 +319,7 @@ export default angular.module('ignite-console', [ .service('CSV', CSV) .service('IGFSs', IGFSs) .service('Models', Models) +.service('Store', Store) // Filters. .filter('byName', byName) .filter('bytes', bytes) diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss b/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss index 3b2d202527ed7..e46e382a44c9c 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss +++ b/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss @@ -18,26 +18,26 @@ connected-clusters { @import "./../../../public/stylesheets/variables.scss"; - div { - position: absolute; - top: 0; - right: 30px; + height: 100%; + display: flex; - display: flex; - align-items: center; - padding: 3px 10px; - - cursor: pointer; - color: white; - font-size: 12px; - line-height: 12px; + .connected-cluster-badge__count { + color: $ignite-brand-success; + margin-left: 5px; + } - background-color: $text-color; + button { + height: 100%; + width: 100%; + background: none !important; + border: none !important; + margin: 0 !important; + padding: 0 !important; + outline: none !important; - border-radius: 0 0 4px 4px; + display: flex; + align-items: center; - [ignite-icon] { - margin-right: 6px; - } + font-size: 14px; } } diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/template.pug b/modules/web-console/frontend/app/components/connected-clusters-badge/template.pug index 49224e5c778c5..dfb4bd9ccf51b 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-badge/template.pug +++ b/modules/web-console/frontend/app/components/connected-clusters-badge/template.pug @@ -14,6 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. -div(ng-click='$ctrl.show()') - svg(ignite-icon='connectedClusters') - | Connected clusters: {{ $ctrl.connectedClusters }} +button(type='button' ng-click='$ctrl.show()') + | My Connected Clusters: {{ $ctrl.connectedClusters }} diff --git a/modules/web-console/frontend/app/components/page-configure/icons/configuration.icon.svg b/modules/web-console/frontend/app/components/page-configure/icons/configuration.icon.svg new file mode 100644 index 0000000000000..7c0ac7aec54c3 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-configure/icons/configuration.icon.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.ts similarity index 91% rename from modules/web-console/frontend/app/components/page-configure/index.js rename to modules/web-console/frontend/app/components/page-configure/index.ts index 531b46cf679ac..dd2ad205dbd65 100644 --- a/modules/web-console/frontend/app/components/page-configure/index.js +++ b/modules/web-console/frontend/app/components/page-configure/index.ts @@ -54,6 +54,10 @@ import 'rxjs/add/operator/skip'; import {Observable} from 'rxjs/Observable'; +import {navigationMenuItem, AppStore} from '../../store'; +import {default as configurationIcon} from './icons/configuration.icon.svg'; +import {default as IconsService} from '../ignite-icon/service'; + Observable.prototype.debug = function(l) { return this.do((v) => console.log(l, v), (e) => console.error(l, e), () => console.log(l, 'completed')); }; @@ -115,21 +119,10 @@ export default angular DefaultState.setRedirectTo(() => 'base.configuration.overview'); }]) .run(registerActivitiesHook) - .run(['ConfigEffects', 'ConfigureState', '$uiRouter', (ConfigEffects, ConfigureState, $uiRouter) => { - $uiRouter.plugin(UIRouterRx); - - if (devTools) { - devTools.subscribe((e) => { - if (e.type === 'DISPATCH' && e.state) ConfigureState.actions$.next(e); - }); - - ConfigureState.actions$ - .filter((e) => e.type !== 'DISPATCH') - .withLatestFrom(ConfigureState.state$.skip(1)) - .subscribe(([action, state]) => devTools.send(action, state)); + .run(['ConfigEffects', 'ConfigureState', '$uiRouter', 'Store', 'IgniteIcon', (ConfigEffects, ConfigureState, $uiRouter, store: AppStore, icons: IconsService) => { + icons.registerIcons({configuration: configurationIcon}); - ConfigureState.addReducer(reduxDevtoolsReducer); - } + $uiRouter.plugin(UIRouterRx); ConfigureState.addReducer(refsReducer({ models: {at: 'domains', store: 'caches'}, @@ -176,6 +169,14 @@ export default angular .do((a) => ConfigureState.dispatchAction(a)) .subscribe(); ConfigEffects.connect(); + + store.dispatch(navigationMenuItem({ + activeSref: 'base.configuration.**', + icon: 'configuration', + label: 'Configuration', + order: 1, + sref: 'base.configuration.overview' + })); }]) .component('pageConfigure', component) .directive('pcIsInCollection', isInCollection) diff --git a/modules/web-console/frontend/app/components/page-forgot-password/controller.js b/modules/web-console/frontend/app/components/page-forgot-password/controller.js index 7d06773c58a72..86597aff4d7c8 100644 --- a/modules/web-console/frontend/app/components/page-forgot-password/controller.js +++ b/modules/web-console/frontend/app/components/page-forgot-password/controller.js @@ -24,22 +24,26 @@ export default class PageForgotPassword { data = {email: null}; /** @type {string} */ serverError = null; + /** @type {JQLite} */ + el; - static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils']; + static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils', '$element']; /** * @param {import('app/modules/user/Auth.service').default} Auth */ - constructor(Auth, IgniteMessages, IgniteFormUtils) { + constructor(Auth, IgniteMessages, IgniteFormUtils, el) { this.Auth = Auth; this.IgniteMessages = IgniteMessages; this.IgniteFormUtils = IgniteFormUtils; + this.el = el; } /** @param {import('./types').IForgotPasswordFormController} form */ canSubmitForm(form) { return form.$error.server ? true : !form.$invalid; } $postLink() { + this.el.addClass('public-page'); this.form.email.$validators.server = () => !this.serverError; } diff --git a/modules/web-console/frontend/app/components/page-forgot-password/run.js b/modules/web-console/frontend/app/components/page-forgot-password/run.js index c7ee6a64a1d3f..d9259e95fc1eb 100644 --- a/modules/web-console/frontend/app/components/page-forgot-password/run.js +++ b/modules/web-console/frontend/app/components/page-forgot-password/run.js @@ -15,6 +15,8 @@ * limitations under the License. */ +import publicTemplate from '../../../views/public.pug'; + /** * @param {import("@uirouter/angularjs").UIRouter} $uiRouter */ @@ -23,7 +25,14 @@ export function registerState($uiRouter) { const state = { name: 'forgotPassword', url: '/forgot-password', - component: 'pageForgotPassword', + views: { + '': { + template: publicTemplate + }, + 'page@forgotPassword': { + component: 'pageForgotPassword' + } + }, unsaved: true, tfMetaTags: { title: 'Forgot Password' diff --git a/modules/web-console/frontend/app/components/page-forgot-password/style.scss b/modules/web-console/frontend/app/components/page-forgot-password/style.scss index 86425c189806e..7893c625697a7 100644 --- a/modules/web-console/frontend/app/components/page-forgot-password/style.scss +++ b/modules/web-console/frontend/app/components/page-forgot-password/style.scss @@ -20,34 +20,22 @@ page-forgot-password { flex-direction: column; flex: 1 0 auto; - section { - margin-left: auto; - margin-right: auto; - width: 530px; - - h3 { - font-size: 38px; - font-weight: 300; - margin: 30px 0 30px; - } - - p { - margin-bottom: 20px; - } + p { + margin-bottom: 20px; + } - .form-field { - margin: 10px 0; - } + .form-field { + margin: 10px 0; + } - .form-footer { - padding: 15px 0; - text-align: right; - display: flex; - align-items: center; + .form-footer { + padding: 15px 0; + text-align: right; + display: flex; + align-items: center; - .btn-ignite { - margin-left: auto; - } + .btn-ignite { + margin-left: auto; } } } diff --git a/modules/web-console/frontend/app/components/page-forgot-password/template.pug b/modules/web-console/frontend/app/components/page-forgot-password/template.pug index 82e7898599873..309e9b93de32a 100644 --- a/modules/web-console/frontend/app/components/page-forgot-password/template.pug +++ b/modules/web-console/frontend/app/components/page-forgot-password/template.pug @@ -16,34 +16,25 @@ include /app/helpers/jade/mixins -web-console-header - web-console-header-left - ignite-header-title - -.container--responsive.body-container - section - - const form = '$ctrl.form' - h3 Forgot password? - p Enter the email address for your account & we'll email you a link to reset your password. - form(name=form novalidate ng-submit='$ctrl.remindPassword()') - +form-field__email({ - label: 'Email:', - model: '$ctrl.data.email', - name: '"email"', - placeholder: 'Input email', - required: true - })( - ng-model-options='{allowInvalid: true}' - autocomplete='email' - ignite-auto-focus - tabindex='0' - ) - +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) - footer.form-footer - a(ui-sref='signin') Back to sign in - button.btn-ignite.btn-ignite--primary( - tabindex='1' - type='submit' - ) Send it to me - -web-console-footer +h3.public-page__title Forgot password? +p Enter the email address for your account & we'll email you a link to reset your password. +form(name='$ctrl.form' novalidate ng-submit='$ctrl.remindPassword()') + +form-field__email({ + label: 'Email:', + model: '$ctrl.data.email', + name: '"email"', + placeholder: 'Input email', + required: true + })( + ng-model-options='{allowInvalid: true}' + autocomplete='email' + ignite-auto-focus + tabindex='0' + ) + +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) + footer.form-footer + a(ui-sref='signin') Back to sign in + button.btn-ignite.btn-ignite--primary( + tabindex='1' + type='submit' + ) Send it to me diff --git a/modules/web-console/frontend/app/components/page-landing/index.js b/modules/web-console/frontend/app/components/page-landing/index.js index 8f678f397c1e4..cabb5bbdef6c1 100644 --- a/modules/web-console/frontend/app/components/page-landing/index.js +++ b/modules/web-console/frontend/app/components/page-landing/index.js @@ -17,6 +17,7 @@ import angular from 'angular'; +import baseTemplate from './public.pug'; import template from './template.pug'; import './style.scss'; @@ -44,7 +45,15 @@ export default angular $stateProvider .state('landing', { url: '/', - template: '', + views: { + '@': { + template: baseTemplate + }, + 'page@landing': { + component: 'pageLanding' + } + }, + // template: '', redirectTo: (trans) => { return trans.injector().get('User').read() .then(() => { diff --git a/modules/web-console/frontend/app/components/page-landing/public.pug b/modules/web-console/frontend/app/components/page-landing/public.pug new file mode 100644 index 0000000000000..d9153f52f2c16 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-landing/public.pug @@ -0,0 +1,21 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +web-console-header(hide-menu-button='true') + .web-console-header-content__title Management console for Apache Ignite + .page-landing__button-signin.btn-ignite.btn-ignite--primary(ui-sref='signin') Sign In +.content(ui-view='page') +web-console-footer.web-console-footer__page-bottom \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/page-landing/style.scss b/modules/web-console/frontend/app/components/page-landing/style.scss index a92206ba27ec6..90f3ddaf7f2e9 100644 --- a/modules/web-console/frontend/app/components/page-landing/style.scss +++ b/modules/web-console/frontend/app/components/page-landing/style.scss @@ -24,8 +24,20 @@ @include custom_btn; } +.page-landing__button-signin { + align-self: center; + margin-left: 30px; + margin-right: var(--page-side-padding) !important; + min-width: 80px; + font-weight: 500; + padding: 10px 25px !important; + flex: 0 0 auto; +} + page-landing { font-family: Roboto; + display: block; + margin: 0 calc(var(--page-side-padding) * -1) calc(var(--page-side-padding) * -1); .btn-custom { @include custom_btn; diff --git a/modules/web-console/frontend/app/components/page-landing/template.pug b/modules/web-console/frontend/app/components/page-landing/template.pug index 0fc1f9c5ffb01..bf926fae07411 100644 --- a/modules/web-console/frontend/app/components/page-landing/template.pug +++ b/modules/web-console/frontend/app/components/page-landing/template.pug @@ -14,12 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. -web-console-header - web-console-header-left - ignite-header-title - web-console-header-right - a#signin_show.btn-ignite.btn-ignite--primary(ui-sref='signin') Sign In - section.intro-container-wrapper .container .col-lg-6.col-md-6.col-sm-6.col-xs-12.intro-content @@ -62,5 +56,3 @@ section.features-container-wrapper p The Web Console allows you to have accounts with different roles. .align-center.text-center a.btn.btn-lg.btn-primary.btn-custom(ui-sref='signup') Get Started - -web-console-footer diff --git a/modules/web-console/frontend/app/components/page-password-changed/controller.ts b/modules/web-console/frontend/app/components/page-password-changed/controller.ts new file mode 100644 index 0000000000000..9696a092fb114 --- /dev/null +++ b/modules/web-console/frontend/app/components/page-password-changed/controller.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {StateService} from '@uirouter/angularjs'; + +export default class implements ng.IPostLink { + static $inject = ['$state', '$timeout', '$element']; + + constructor($state: StateService, $timeout: ng.ITimeoutService, private el: JQLite) { + $timeout(() => { + $state.go('signin'); + }, 10000); + } + + $postLink() { + this.el.addClass('public-page'); + } +} diff --git a/modules/web-console/frontend/app/components/page-password-changed/index.js b/modules/web-console/frontend/app/components/page-password-changed/index.js index 7c244beff0622..903138f0b1ef1 100644 --- a/modules/web-console/frontend/app/components/page-password-changed/index.js +++ b/modules/web-console/frontend/app/components/page-password-changed/index.js @@ -19,6 +19,7 @@ import angular from 'angular'; import template from './template.pug'; import controller from './controller'; +import publicTemplate from '../../../views/public.pug'; import './style.scss'; @@ -32,7 +33,14 @@ export default angular .config(['$stateProvider', ($stateProvider) => { $stateProvider.state('password.send', { url: '/changed', - component: 'pagePasswordChanged', + views: { + '@': { + template: publicTemplate + }, + 'page@password.send': { + component: 'pagePasswordChanged' + } + }, tfMetaTags: { title: 'Password send' }, diff --git a/modules/web-console/frontend/app/components/page-password-changed/style.scss b/modules/web-console/frontend/app/components/page-password-changed/style.scss index eeb44ec451eab..e32e9b2c4849c 100644 --- a/modules/web-console/frontend/app/components/page-password-changed/style.scss +++ b/modules/web-console/frontend/app/components/page-password-changed/style.scss @@ -19,17 +19,10 @@ page-password-changed { display: flex; flex: 1 0 auto; flex-direction: column; - min-height: 100%; - - font-family: Roboto; - - .body-container { - display: flex; - flex-direction: column; - justify-content: center; - align-items: center; - } + justify-content: center; + align-items: center; + max-width: initial !important; h2 { margin-bottom: 30px; diff --git a/modules/web-console/frontend/app/components/page-password-changed/template.pug b/modules/web-console/frontend/app/components/page-password-changed/template.pug index 7d2d986c812b2..82c2cc1608ac7 100644 --- a/modules/web-console/frontend/app/components/page-password-changed/template.pug +++ b/modules/web-console/frontend/app/components/page-password-changed/template.pug @@ -14,9 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. -.container.body-container - h2 Ready! +h2 Ready! - p - | Further instructions for password reset have been sent to your e-mail address.#[br] - | You'll be redirected back automatically in a few seconds. If not, please #[a(ui-sref='signin') click here]. +p + | Further instructions for password reset have been sent to your e-mail address.#[br] + | You'll be redirected back automatically in a few seconds. If not, please #[a(ui-sref='signin') click here]. diff --git a/modules/web-console/frontend/app/components/page-password-reset/controller.js b/modules/web-console/frontend/app/components/page-password-reset/controller.js index 53d6c69393a95..b69e56610a7b3 100644 --- a/modules/web-console/frontend/app/components/page-password-reset/controller.js +++ b/modules/web-console/frontend/app/components/page-password-reset/controller.js @@ -16,7 +16,9 @@ */ export default class { - static $inject = ['$modal', '$http', '$state', 'IgniteMessages']; + static $inject = ['$modal', '$http', '$state', 'IgniteMessages', '$element']; + /** @type {JQLite} */ + el; /** * @param {mgcrea.ngStrap.modal.IModalService} $modal @@ -24,10 +26,15 @@ export default class { * @param {import('@uirouter/angularjs').StateService} $state * @param {ReturnType} Messages */ - constructor($modal, $http, $state, Messages) { + constructor($modal, $http, $state, Messages, el) { this.$http = $http; this.$state = $state; this.Messages = Messages; + this.el = el; + } + + $postLink() { + this.el.addClass('public-page'); } $onInit() { diff --git a/modules/web-console/frontend/app/components/page-password-reset/index.js b/modules/web-console/frontend/app/components/page-password-reset/index.js index e1042a6565c56..c07fa23eb7234 100644 --- a/modules/web-console/frontend/app/components/page-password-reset/index.js +++ b/modules/web-console/frontend/app/components/page-password-reset/index.js @@ -22,6 +22,7 @@ import _ from 'lodash'; import template from './template.pug'; import controller from './controller'; +import publicTemplate from '../../../views/public.pug'; import './style.scss'; @@ -42,7 +43,14 @@ export default angular }) .state('password.reset', { url: '/reset?{token}', - component: 'pagePasswordReset', + views: { + '@': { + template: publicTemplate + }, + 'page@password.reset': { + component: 'pagePasswordReset' + } + }, redirectTo: (trans) => { if (_.isEmpty(trans.params('to').token)) return 'signin'; diff --git a/modules/web-console/frontend/app/components/page-password-reset/style.scss b/modules/web-console/frontend/app/components/page-password-reset/style.scss index 75f758e676048..05ef953114218 100644 --- a/modules/web-console/frontend/app/components/page-password-reset/style.scss +++ b/modules/web-console/frontend/app/components/page-password-reset/style.scss @@ -20,12 +20,19 @@ page-password-reset { flex: 1 0 auto; flex-direction: column; - footer { + .form-footer { + padding: 15px 0; + text-align: right; display: flex; - justify-content: flex-end; + align-items: center; + + .btn-ignite { + margin-left: auto; + } } - .btn-ignite + .btn-ignite { - margin-left: 10px; + form { + display: grid; + grid-gap: 10px; } } diff --git a/modules/web-console/frontend/app/components/page-password-reset/template.pug b/modules/web-console/frontend/app/components/page-password-reset/template.pug index 1acfab82eb78f..c0df3c9bb27c6 100644 --- a/modules/web-console/frontend/app/components/page-password-reset/template.pug +++ b/modules/web-console/frontend/app/components/page-password-reset/template.pug @@ -16,65 +16,45 @@ include /app/helpers/jade/mixins -web-console-header - web-console-header-left - ignite-header-title +//- This doesn't seem to do anything 😵 +.main-content(ng-if='error') + .text-center + p {{::$ctrl.ui.error}} +h3.public-page__title(ng-if-start='$ctrl.ui.token && !$ctrl.ui.error') Reset Password +form.page-password-reset__grid(name='$ctrl.form' ng-init='reset_info.token = token' ng-if-end) + +form-field__email({ + label: 'E-mail:', + model: '$ctrl.ui.email', + disabled: true + }) + + +form-field__password({ + label: 'New password:', + model: '$ctrl.ui.password', + name: '"password"', + required: true, + placeholder: 'New password' + })( + ignite-auto-focus + ignite-on-enter-focus-move='passwordConfirmInput' + ) + +form-field__password({ + label: 'Confirm password:', + model: 'confirm', + name: '"passwordConfirm"', + required: true, + placeholder: 'Confirm new password' + })( + ignite-on-enter-focus-move='resetForm.$valid && resetPassword(user_info)' + ignite-match='$ctrl.ui.password' + ) + + footer.form-footer + a(ui-sref='default-state') Cancel + button.btn-ignite.btn-ignite--primary( + ng-disabled='$ctrl.form.$invalid' + ng-click='$ctrl.resetPassword()' + ) + svg.icon-left(ignite-icon='checkmark') + | Save Changes -.container.body-container - .row - .main-content(ng-if='error') - .text-center - p {{::$ctrl.ui.error}} - div(ng-if='$ctrl.ui.token && !$ctrl.ui.error') - header.header-with-selector - div - h1 Reset Password - - -var form = '$ctrl.form' - form.theme--ignite(name='$ctrl.form' ng-init='reset_info.token = token') - .row - .col-50 - +form-field__email({ - label: 'E-mail:', - model: '$ctrl.ui.email', - disabled: true - }) - - .row - .col-50 - +form-field__password({ - label: 'New password:', - model: '$ctrl.ui.password', - name: '"password"', - required: true, - placeholder: 'New password' - })( - ignite-auto-focus - ignite-on-enter-focus-move='passwordConfirmInput' - ) - - .row - .col-50 - +form-field__password({ - label: 'Confirm password:', - model: 'confirm', - name: '"passwordConfirm"', - required: true, - placeholder: 'Confirm new password' - })( - ignite-on-enter-focus-move='resetForm.$valid && resetPassword(user_info)' - ignite-match='$ctrl.ui.password' - ) - - hr - - footer - a.btn-ignite.btn-ignite--link-success(type='button' ui-sref='default-state') Cancel - button.btn-ignite.btn-ignite--success( - ng-disabled='$ctrl.form.$invalid' - ng-click='$ctrl.resetPassword()' - ) - svg.icon-left(ignite-icon='checkmark') - | Save Changes - -web-console-footer diff --git a/modules/web-console/frontend/app/components/page-queries/index.js b/modules/web-console/frontend/app/components/page-queries/index.ts similarity index 91% rename from modules/web-console/frontend/app/components/page-queries/index.js rename to modules/web-console/frontend/app/components/page-queries/index.ts index ba9e5114dd0c8..2e25cb0dad337 100644 --- a/modules/web-console/frontend/app/components/page-queries/index.js +++ b/modules/web-console/frontend/app/components/page-queries/index.ts @@ -24,6 +24,7 @@ import queriesNotebook from './components/queries-notebook'; import pageQueriesCmp from './component'; import {default as ActivitiesData} from 'app/core/activities/Activities.data'; import Notebook from './notebook.service'; +import {navigationMenuItem, AppStore} from '../../store'; /** * @param {uirouter.UIRouter} $uiRouter @@ -42,6 +43,15 @@ export default angular.module('ignite-console.sql', [ queriesNotebooksList.name, queriesNotebook.name ]) + .run(['Store', (store: AppStore) => { + store.dispatch(navigationMenuItem({ + activeSref: 'base.sql.**', + icon: 'sql', + label: 'Queries', + order: 2, + sref: 'base.sql.tabs.notebooks-list' + })); + }]) .component('pageQueries', pageQueriesCmp) .component('pageQueriesSlot', { require: { diff --git a/modules/web-console/frontend/app/components/page-signin/controller.ts b/modules/web-console/frontend/app/components/page-signin/controller.ts index c461b86e92f0d..18fee0bc5feda 100644 --- a/modules/web-console/frontend/app/components/page-signin/controller.ts +++ b/modules/web-console/frontend/app/components/page-signin/controller.ts @@ -27,7 +27,7 @@ interface ISigninFormController extends ng.IFormController { password: ng.INgModelController } -export default class { +export default class implements ng.IPostLink { data: ISiginData = { email: null, password: null @@ -37,15 +37,16 @@ export default class { serverError: string = null; - static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils']; + static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils', '$element']; - constructor(private Auth: AuthService, private IgniteMessages, private IgniteFormUtils) {} + constructor(private Auth: AuthService, private IgniteMessages, private IgniteFormUtils, private el: JQLite) {} canSubmitForm(form: ISigninFormController) { return form.$error.server ? true : !form.$invalid; } $postLink() { + this.el.addClass('public-page'); this.form.email.$validators.server = () => !this.serverError; this.form.password.$validators.server = () => !this.serverError; } diff --git a/modules/web-console/frontend/app/components/page-signin/run.ts b/modules/web-console/frontend/app/components/page-signin/run.ts index aa5647c44a363..4c0a1e6ced70e 100644 --- a/modules/web-console/frontend/app/components/page-signin/run.ts +++ b/modules/web-console/frontend/app/components/page-signin/run.ts @@ -15,6 +15,7 @@ * limitations under the License. */ +import publicTemplate from '../../../views/public.pug'; import {UIRouter} from '@uirouter/angularjs'; import {IIgniteNg1StateDeclaration} from 'app/types'; @@ -22,7 +23,14 @@ export function registerState($uiRouter: UIRouter) { const state: IIgniteNg1StateDeclaration = { url: '/signin', name: 'signin', - component: 'pageSignin', + views: { + '': { + template: publicTemplate + }, + 'page@signin': { + component: 'pageSignin' + } + }, unsaved: true, redirectTo: (trans) => { const skipStates = new Set(['signup', 'forgotPassword', 'landing']); diff --git a/modules/web-console/frontend/app/components/page-signin/style.scss b/modules/web-console/frontend/app/components/page-signin/style.scss index 51da101770161..729d439dbb055 100644 --- a/modules/web-console/frontend/app/components/page-signin/style.scss +++ b/modules/web-console/frontend/app/components/page-signin/style.scss @@ -18,32 +18,19 @@ page-signin { display: flex; flex-direction: column; - flex: 1 0 auto; - section { - margin-left: auto; - margin-right: auto; - width: 530px; - - h3 { - font-size: 38px; - font-weight: 300; - margin: 30px 0 30px; - } - - .form-field { - margin: 10px 0; - } + .form-field { + margin: 10px 0; + } - .form-footer { - padding: 15px 0; - text-align: right; - display: flex; - align-items: center; + .form-footer { + padding: 15px 0; + text-align: right; + display: flex; + align-items: center; - .btn-ignite { - margin-left: auto; - } + .btn-ignite { + margin-left: auto; } } diff --git a/modules/web-console/frontend/app/components/page-signin/template.pug b/modules/web-console/frontend/app/components/page-signin/template.pug index b31a9bdb69798..2a128a0329111 100644 --- a/modules/web-console/frontend/app/components/page-signin/template.pug +++ b/modules/web-console/frontend/app/components/page-signin/template.pug @@ -16,44 +16,35 @@ include /app/helpers/jade/mixins -web-console-header - web-console-header-left - ignite-header-title - -.container--responsive.body-container - section - -var form = '$ctrl.form' - h3 Sign In - form(name=form novalidate ng-submit='$ctrl.signin()') - +form-field__email({ - label: 'Email:', - model: '$ctrl.data.email', - name: '"email"', - placeholder: 'Input email', - required: true - })( - ng-model-options='{allowInvalid: true}' - autocomplete='email' - ignite-auto-focus - ) - +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) - +form-field__password({ - label: 'Password:', - model: '$ctrl.data.password', - name: '"password"', - placeholder: 'Input password', - required: true - })( - ng-model-options='{allowInvalid: true}' - autocomplete='current-password' - ) - +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) - footer.form-footer - a(ui-sref='forgotPassword({email: $ctrl.data.email})') Forgot password? - button.btn-ignite.btn-ignite--primary( - type='submit' - ) Sign In - footer.page-signin__no-account-message - | Don't have an account? #[a(ui-sref='signup') Get started] - -web-console-footer +h3.public-page__title Sign In +form(name='$ctrl.form' novalidate ng-submit='$ctrl.signin()') + +form-field__email({ + label: 'Email:', + model: '$ctrl.data.email', + name: '"email"', + placeholder: 'Input email', + required: true + })( + ng-model-options='{allowInvalid: true}' + autocomplete='email' + ignite-auto-focus + ) + +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) + +form-field__password({ + label: 'Password:', + model: '$ctrl.data.password', + name: '"password"', + placeholder: 'Input password', + required: true + })( + ng-model-options='{allowInvalid: true}' + autocomplete='current-password' + ) + +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`}) + footer.form-footer + a(ui-sref='forgotPassword({email: $ctrl.data.email})') Forgot password? + button.btn-ignite.btn-ignite--primary( + type='submit' + ) Sign In +footer.page-signin__no-account-message + | Don't have an account? #[a(ui-sref='signup') Get started] diff --git a/modules/web-console/frontend/app/components/page-signup/controller.ts b/modules/web-console/frontend/app/components/page-signup/controller.ts index ddf77be099ca9..93d99870f3981 100644 --- a/modules/web-console/frontend/app/components/page-signup/controller.ts +++ b/modules/web-console/frontend/app/components/page-signup/controller.ts @@ -20,7 +20,7 @@ import MessagesFactory from '../../services/Messages.service'; import FormUtilsFactoryFactory from '../../services/FormUtils.service'; import {ISignupData} from '../form-signup'; -export default class PageSignup { +export default class PageSignup implements ng.IPostLink { form: ng.IFormController; data: ISignupData = { @@ -34,14 +34,19 @@ export default class PageSignup { serverError: string | null = null; - static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils']; + static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils', '$element']; constructor( private Auth: Auth, private IgniteMessages: ReturnType, - private IgniteFormUtils: ReturnType + private IgniteFormUtils: ReturnType, + private el: JQLite ) {} + $postLink() { + this.el.addClass('public-page'); + } + canSubmitForm(form: PageSignup['form']) { return form.$error.server ? true : !form.$invalid; } diff --git a/modules/web-console/frontend/app/components/page-signup/run.js b/modules/web-console/frontend/app/components/page-signup/run.js index 1d04fa257714f..bd2763802e85e 100644 --- a/modules/web-console/frontend/app/components/page-signup/run.js +++ b/modules/web-console/frontend/app/components/page-signup/run.js @@ -15,6 +15,8 @@ * limitations under the License. */ +import publicTemplate from '../../../views/public.pug'; + /** * @param {import("@uirouter/angularjs").UIRouter} $uiRouter */ @@ -23,7 +25,14 @@ export function registerState($uiRouter) { const state = { name: 'signup', url: '/signup', - component: 'pageSignup', + views: { + '': { + template: publicTemplate + }, + 'page@signup': { + component: 'pageSignup' + } + }, unsaved: true, tfMetaTags: { title: 'Sign Up' diff --git a/modules/web-console/frontend/app/components/page-signup/style.scss b/modules/web-console/frontend/app/components/page-signup/style.scss index 98c4729bf5ad5..0930036fb8e14 100644 --- a/modules/web-console/frontend/app/components/page-signup/style.scss +++ b/modules/web-console/frontend/app/components/page-signup/style.scss @@ -20,26 +20,14 @@ page-signup { flex-direction: column; flex: 1 0 auto; - section { - h3 { - font-size: 38px; - font-weight: 300; - margin: 30px 0 30px; - } - - margin-left: auto; - margin-right: auto; - width: 530px; - - form footer { - padding: 15px 0; - text-align: right; - display: flex; - align-items: center; + form footer { + padding: 15px 0; + text-align: right; + display: flex; + align-items: center; - .btn-ignite { - margin-left: auto; - } + .btn-ignite { + margin-left: auto; } } diff --git a/modules/web-console/frontend/app/components/page-signup/template.pug b/modules/web-console/frontend/app/components/page-signup/template.pug index 5584dfcb67129..686079ad94a1b 100644 --- a/modules/web-console/frontend/app/components/page-signup/template.pug +++ b/modules/web-console/frontend/app/components/page-signup/template.pug @@ -14,26 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. -include /app/helpers/jade/mixins - -web-console-header - web-console-header-left - ignite-header-title - -.container--responsive.body-container - section - h3 Don't Have An Account? - form(name='$ctrl.form' novalidate ng-submit='$ctrl.signup()') - form-signup( - outer-form='$ctrl.form' - ng-model='$ctrl.data' - server-error='$ctrl.serverError' - ) - footer.full-width.form-footer - button.btn-ignite.btn-ignite--primary( - type='submit' - ) Sign Up - footer.page-signup__has-account-message - | Already have an account? #[a(ui-sref='signin') Sign in here] - -web-console-footer +h3.public-page__title Don't Have An Account? +form(name='$ctrl.form' novalidate ng-submit='$ctrl.signup()') + form-signup( + outer-form='$ctrl.form' + ng-model='$ctrl.data' + server-error='$ctrl.serverError' + ) + footer.full-width.form-footer + button.btn-ignite.btn-ignite--primary( + type='submit' + ) Sign Up +footer.page-signup__has-account-message + | Already have an account? #[a(ui-sref='signin') Sign in here] diff --git a/modules/web-console/frontend/app/components/permanent-notifications/component.ts b/modules/web-console/frontend/app/components/permanent-notifications/component.ts new file mode 100644 index 0000000000000..e86d3e512713a --- /dev/null +++ b/modules/web-console/frontend/app/components/permanent-notifications/component.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import controller from './controller'; +import './style.scss'; + +export const component: ng.IComponentOptions = { + template, + controller +}; diff --git a/modules/web-console/frontend/app/modules/navbar/navbar.directive.js b/modules/web-console/frontend/app/components/permanent-notifications/controller.ts similarity index 71% rename from modules/web-console/frontend/app/modules/navbar/navbar.directive.js rename to modules/web-console/frontend/app/components/permanent-notifications/controller.ts index e400cf7d2f54a..d67230431b311 100644 --- a/modules/web-console/frontend/app/modules/navbar/navbar.directive.js +++ b/modules/web-console/frontend/app/components/permanent-notifications/controller.ts @@ -15,18 +15,14 @@ * limitations under the License. */ -export default function factory(IgniteNavbar) { - function controller() { - const ctrl = this; - - ctrl.items = IgniteNavbar; +export default class PermanentNotifications { + static $inject = ['UserNotifications', '$rootScope', '$window'] + constructor( + private UserNotifications: unknown, + private $rootScope: ng.IRootScopeService, + private $window: ng.IWindowService + ) {} + closeDemo() { + this.$window.close(); } - - return { - restrict: 'A', - controller, - controllerAs: 'navbar' - }; } - -factory.$inject = ['IgniteNavbar']; diff --git a/modules/web-console/frontend/app/components/permanent-notifications/index.ts b/modules/web-console/frontend/app/components/permanent-notifications/index.ts new file mode 100644 index 0000000000000..33a9189132002 --- /dev/null +++ b/modules/web-console/frontend/app/components/permanent-notifications/index.ts @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {component} from './component'; + +export default angular.module('ignite-console.permanent-notifications', []) + .component('permanentNotifications', component); diff --git a/modules/web-console/frontend/app/components/permanent-notifications/style.scss b/modules/web-console/frontend/app/components/permanent-notifications/style.scss new file mode 100644 index 0000000000000..e40f21a359630 --- /dev/null +++ b/modules/web-console/frontend/app/components/permanent-notifications/style.scss @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +permanent-notifications { + display: block; + + @import "./../../../public/stylesheets/variables.scss"; + + .wch-notification { + line-height: 16px; + padding: 7px; + background: $brand-warning; + font-size: 16px; + text-align: center; + + a { + color: $brand-info; + } + + &+.wch-notification { + border-top: 1px solid darken($brand-warning, 15%); + } + } + + .wch-notification.wch-notification--demo { + color: white; + background: $ignite-brand-success; + border: none; + + a { + color: white; + text-decoration: underline; + + &:hover { + color: #ffab40; + text-decoration: none; + } + } + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/permanent-notifications/template.pug b/modules/web-console/frontend/app/components/permanent-notifications/template.pug new file mode 100644 index 0000000000000..6abe61181a512 --- /dev/null +++ b/modules/web-console/frontend/app/components/permanent-notifications/template.pug @@ -0,0 +1,23 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.wch-notification(ng-show='$ctrl.UserNotifications.isShown && $ctrl.UserNotifications.message' ng-bind-html='$ctrl.UserNotifications.message') + +.wch-notification(ng-show='$ctrl.$rootScope.user.becomeUsed') + | You are currently viewing user #[strong {{$ctrl.$rootScope.user.firstName}} {{$ctrl.$rootScope.user.lastName}}] as administrator. #[a(ng-click='$ctrl.$rootScope.revertIdentity()') Revert to your identity?] + +.wch-notification.wch-notification--demo(ng-if='$ctrl.$rootScope.IgniteDemoMode') + | You are now in #[b Demo Mode]. #[a(ng-click='$ctrl.closeDemo()') Close Demo?] diff --git a/modules/web-console/frontend/app/components/timed-redirection/style.scss b/modules/web-console/frontend/app/components/timed-redirection/style.scss index 5b0ecfd417f6c..487fa2d8d2486 100644 --- a/modules/web-console/frontend/app/components/timed-redirection/style.scss +++ b/modules/web-console/frontend/app/components/timed-redirection/style.scss @@ -23,6 +23,7 @@ timed-redirection { display: flex; flex-direction: column; flex: 1; + min-height: 100%; .timed-redirection--wrapper { text-align: center; diff --git a/modules/web-console/frontend/app/components/web-console-footer/component.js b/modules/web-console/frontend/app/components/web-console-footer/component.js index 40a1051a75820..bb2f7f7728cd0 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/component.js +++ b/modules/web-console/frontend/app/components/web-console-footer/component.js @@ -17,7 +17,9 @@ import template from './template.pug'; import './style.scss'; +import controller from './controller'; export default { - template + template, + controller }; diff --git a/modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/style.scss b/modules/web-console/frontend/app/components/web-console-footer/controller.ts similarity index 70% rename from modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/style.scss rename to modules/web-console/frontend/app/components/web-console-footer/controller.ts index 47b88af82c7c3..0811748f075d6 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/style.scss +++ b/modules/web-console/frontend/app/components/web-console-footer/controller.ts @@ -1,4 +1,4 @@ -/*! +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -15,17 +15,13 @@ * limitations under the License. */ -web-console-footer-links { - margin-left: auto; +import {default as Version} from '../../services/Version.service'; - a + a { - margin-left: 5px; - - &:before { - content: '|'; - display: inline-block; - margin-right: 5px; - color: #fafafa; - } +export default class WebConsoleFooter { + static $inject = ['IgniteVersion', '$rootScope'] + constructor(private Version: Version, private $root: ng.IRootScopeService) {} + year = new Date().getFullYear() + get userIsAuthorized() { + return !!this.$root.user; } } diff --git a/modules/web-console/frontend/app/components/web-console-footer/index.js b/modules/web-console/frontend/app/components/web-console-footer/index.js index e26b60238a6bb..4fa461779da83 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/index.js +++ b/modules/web-console/frontend/app/components/web-console-footer/index.js @@ -17,9 +17,16 @@ import angular from 'angular'; import component from './component'; -import componentLinks from './components/web-console-footer-links/component'; export default angular .module('ignite-console.web-console-footer', []) .component('webConsoleFooter', component) - .component('webConsoleFooterLinks', componentLinks); + .directive('webConsoleFooterPageBottom', function() { + return { + restrict: 'C', + link(scope, el) { + el.parent().addClass('wrapper-public'); + scope.$on('$destroy', () => el.parent().removeClass('wrapper-public')); + } + }; + }); diff --git a/modules/web-console/frontend/app/components/web-console-footer/style.scss b/modules/web-console/frontend/app/components/web-console-footer/style.scss index f3de59fccedb5..fe5eb64e584b0 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/style.scss +++ b/modules/web-console/frontend/app/components/web-console-footer/style.scss @@ -16,46 +16,87 @@ */ web-console-footer { - display: block; - color: #fafafa; - background: linear-gradient(to bottom, #5f5e5e, #393939); - font-family: Roboto; font-size: 12px; - font-weight: 300; - padding-top: 12px; - padding-bottom: 12px; - p { - margin-bottom: 0; + .#{&}__legal { + color: rgba(0, 0, 0, 0.54); + } + + &.web-console-footer__sidebar-closed { + width: var(--width-narrow); + border-top: 1px solid #dddddd; + + a { + color: var(--inactive-link-color); + margin-left: auto; + margin-right: auto; + width: 40px; + height: 40px; + display: inline-flex; + align-items: center; + justify-content: center; - &:first-of-type { - margin-bottom: 5px; + &:hover, &:focus { + color: var(--active-link-color); + } + [ignite-icon] { + transform: scale(1.25); + } + } + + .web-console-footer__sidebar-stuff { + position: sticky; + display: grid; + grid-row-gap: 15px; + padding-top: 20px; + padding-bottom: 20px; + } + + .web-console-footer__regular-stuff { + display: none; } } - .wcf-content { + &.web-console-footer__sidebar-opened { + width: var(--width-wide); display: flex; - flex-direction: row; - align-items: center; + flex-direction: column; + padding: 27px 18px 27px 27px; - & > :nth-child(2) { - margin-left: auto; + .web-console-footer__sidebar-stuff { + display: none; + } + + .web-console-footer__regular-stuff { + display: block; } } +} + +.web-console-footer__page-bottom { + grid-area: footer; + color: #fafafa; + background: #393939; + font-size: 12px; + font-weight: 300; + padding: 12px var(--page-side-padding); + + .web-console-footer__sidebar-stuff { + display: none; + } - ignite-powered-by-apache { - & > a { - margin-left: 40px; - } + p { + margin: 0; } a { - color: #ee8e89; - - &:hover, &.hover, - &:focus, &.focus { - color: #ee2b27; - text-decoration: none; + color: #ee8e89 !important; + &:hover, &:focus { + color: #ee2b27 !important; } } + + .web-console-footer__legal { + color: inherit; + } } diff --git a/modules/web-console/frontend/app/components/web-console-footer/template.pug b/modules/web-console/frontend/app/components/web-console-footer/template.pug index aa2812af64904..2c1495e342e09 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/template.pug +++ b/modules/web-console/frontend/app/components/web-console-footer/template.pug @@ -14,7 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. -.container--responsive.wcf-content - ignite-footer - web-console-footer-links(ng-if='$parent.user') - ignite-powered-by-apache \ No newline at end of file + +.web-console-footer__sidebar-stuff + a( + href="/api/v1/downloads/agent" + target="_self" + ng-if='$ctrl.userIsAuthorized' + title='Download Agent' + ) + svg(ignite-icon='downloadAgent') + +.web-console-footer__regular-stuff + p Apache Ignite Web Console ({{$ctrl.Version.webConsole}}) + p © {{::$ctrl.year}} The Apache Software Foundation. + p.web-console-footer__legal Apache, Apache Ignite, the Apache feather and the Apache Ignite logo are trademarks of The Apache Software Foundation. + + a(href="/api/v1/downloads/agent" target="_self" ng-if='$ctrl.userIsAuthorized') Download Agent \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-header/component.js b/modules/web-console/frontend/app/components/web-console-header/component.ts similarity index 66% rename from modules/web-console/frontend/app/components/web-console-header/component.js rename to modules/web-console/frontend/app/components/web-console-header/component.ts index 02b30801e613b..ad7e8a78d0630 100644 --- a/modules/web-console/frontend/app/components/web-console-header/component.js +++ b/modules/web-console/frontend/app/components/web-console-header/component.ts @@ -17,37 +17,27 @@ import template from './template.pug'; import './style.scss'; +import {AppStore, toggleSidebar} from '../../store'; export default { template, controller: class { - static $inject = ['$rootScope', '$scope', '$state', 'IgniteBranding', 'UserNotifications']; + static $inject = ['$rootScope', '$scope', '$state', 'IgniteBranding', 'UserNotifications', 'Store']; - static connectedClustersUnvisibleStates = [ - '403', '404', 'signin' - ]; - - constructor($rootScope, $scope, $state, branding, UserNotifications) { + constructor($rootScope, $scope, $state, branding, UserNotifications, private store: AppStore) { Object.assign(this, {$rootScope, $scope, $state, branding, UserNotifications}); } - setConnectedClustersVisible() { - this.isConnectedClustersVisible = - !this.constructor.connectedClustersUnvisibleStates.some((state) => this.$state.includes(state)); + toggleSidebar() { + this.store.dispatch(toggleSidebar()); } isAuthorized() { return !!this.$rootScope.user; } - - $onInit() { - this.setConnectedClustersVisible(); - - this.$scope.$on('$stateChangeSuccess', () => this.setConnectedClustersVisible()); - } }, - transclude: { - slotLeft: '?webConsoleHeaderLeft', - slotRight: '?webConsoleHeaderRight' + transclude: true, + bindings: { + hideMenuButton: ' + ) {} + + private _openTab(stateName: string) { + this.$window.open(this.$state.href(stateName, {}), '_blank'); + } + + startDemo() { + if (!this.$root.user.demoCreated) + return this._openTab('demo.reset'); + + this.Confirm.confirm('Would you like to continue with previous demo session?', true, false) + .then((resume) => { + if (resume) + return this._openTab('demo.resume'); + + this._openTab('demo.reset'); + }); + } +} diff --git a/modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/template.pug b/modules/web-console/frontend/app/components/web-console-header/components/demo-mode-button/template.pug similarity index 90% rename from modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/template.pug rename to modules/web-console/frontend/app/components/web-console-header/components/demo-mode-button/template.pug index 5c0446786bb26..f566ac4c65fdd 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/template.pug +++ b/modules/web-console/frontend/app/components/web-console-header/components/demo-mode-button/template.pug @@ -14,4 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. -a(href="/api/v1/downloads/agent" target="_self") Download Agent + +button.btn-ignite.btn-ignite--success( + ng-click='$ctrl.startDemo()' +) Start Demo \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/component.js b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/component.ts similarity index 92% rename from modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/component.js rename to modules/web-console/frontend/app/components/web-console-header/components/user-menu/component.ts index 40a1051a75820..f6141d9baf2b7 100644 --- a/modules/web-console/frontend/app/components/web-console-footer/components/web-console-footer-links/component.js +++ b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/component.ts @@ -16,8 +16,10 @@ */ import template from './template.pug'; +import controller from './controller'; import './style.scss'; export default { - template + template, + controller }; diff --git a/modules/web-console/frontend/app/modules/navbar/userbar.directive.js b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/controller.ts similarity index 50% rename from modules/web-console/frontend/app/modules/navbar/userbar.directive.js rename to modules/web-console/frontend/app/components/web-console-header/components/user-menu/controller.ts index b636a34d82ff5..812d3b54ae3e9 100644 --- a/modules/web-console/frontend/app/modules/navbar/userbar.directive.js +++ b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/controller.ts @@ -15,45 +15,39 @@ * limitations under the License. */ -/** - * @param {ng.IRootScopeService} $root - * @param {unknown} IgniteUserbar - * @param {unknown} AclService - */ -function controller($root, IgniteUserbar, AclService) { - const ctrl = this; - - this.$onInit = () => { - ctrl.items = [ +export default class UserMenu { + static $inject = ['$rootScope', 'IgniteUserbar', 'AclService', '$state', 'gettingStarted'] + constructor( + private $root: ng.IRootScopeService, + private IgniteUserbar: any, + private AclService: any, + private $state: any, + private gettingStarted: any + ) {} + $onInit() { + this.items = [ {text: 'Profile', sref: 'base.settings.profile'}, - {text: 'Getting started', click: 'gettingStarted.tryShow(true)'} + {text: 'Getting started', click: '$ctrl.gettingStarted.tryShow(true)'} ]; const _rebuildSettings = () => { - ctrl.items.splice(2); + this.items.splice(2); - if (AclService.can('admin_page')) - ctrl.items.push({text: 'Admin panel', sref: 'base.settings.admin'}); + if (this.AclService.can('admin_page')) + this.items.push({text: 'Admin panel', sref: 'base.settings.admin'}); - ctrl.items.push(...IgniteUserbar); + this.items.push(...this.IgniteUserbar); - if (AclService.can('logout')) - ctrl.items.push({text: 'Log out', sref: 'logout'}); + if (this.AclService.can('logout')) + this.items.push({text: 'Log out', sref: 'logout'}); }; - if ($root.user) - _rebuildSettings(null, $root.user); - - $root.$on('user', _rebuildSettings); - }; -} - -controller.$inject = ['$rootScope', 'IgniteUserbar', 'AclService']; + if (this.$root.user) + _rebuildSettings(null, this.$root.user); -export default function() { - return { - restrict: 'A', - controller, - controllerAs: 'userbar' - }; + this.$root.$on('user', _rebuildSettings); + } + get user() { + return this.$root.user; + } } diff --git a/modules/web-console/frontend/app/components/page-password-changed/controller.js b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/style.scss similarity index 81% rename from modules/web-console/frontend/app/components/page-password-changed/controller.js rename to modules/web-console/frontend/app/components/web-console-header/components/user-menu/style.scss index e3b0a035b5e14..898047b52a9ea 100644 --- a/modules/web-console/frontend/app/components/page-password-changed/controller.js +++ b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/style.scss @@ -15,12 +15,14 @@ * limitations under the License. */ -export default class { - static $inject = ['$state', '$timeout']; +user-menu { + --active-link-color: #ee2b27; + font-size: 14px; - constructor($state, $timeout) { - $timeout(() => { - $state.go('signin'); - }, 10000); + &>.active { + color: var(--active-link-color) } -} + .caret { + margin-left: 8px; + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-header/components/user-menu/template.pug b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/template.pug new file mode 100644 index 0000000000000..e946c78a9b9ba --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-header/components/user-menu/template.pug @@ -0,0 +1,26 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +div( + ng-class='{active: $ctrl.$state.includes("base.settings")}' + ng-click='$event.stopPropagation()' + bs-dropdown='$ctrl.items' + data-placement='bottom-right' + data-trigger='hover focus' + data-container='self' +) + span {{$ctrl.user.firstName}} {{$ctrl.user.lastName}} + span.caret \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/component.ts b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/component.ts new file mode 100644 index 0000000000000..e86d3e512713a --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/component.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import controller from './controller'; +import './style.scss'; + +export const component: ng.IComponentOptions = { + template, + controller +}; diff --git a/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/controller.ts b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/controller.ts new file mode 100644 index 0000000000000..4345553d36b53 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/controller.ts @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {StateService} from '@uirouter/angularjs'; + +export default class WebConsoleHeaderContent { + static $inject = ['$rootScope', '$state'] + constructor( + private $rootScope: ng.IRootScopeService, + private $state: StateService + ) {} + + static connectedClusterInvisibleStates = [ + '403', '404', 'signin' + ]; + + get showConnectedClusters(): boolean { + return this.$rootScope.user && + !this.$rootScope.IgniteDemoMode && + !this.constructor.connectedClusterInvisibleStates.some((state) => this.$state.includes(state)) && + !this.$rootScope.user.becomeUsed; + } + + get showUserMenu(): boolean { + return !!this.$rootScope.user; + } + + get showDemoModeButton(): boolean { + return this.$rootScope.user && !this.$rootScope.user.becomeUsed && !this.$rootScope.IgniteDemoMode; + } +} diff --git a/modules/web-console/frontend/app/modules/branding/header-title.directive.js b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/style.scss similarity index 60% rename from modules/web-console/frontend/app/modules/branding/header-title.directive.js rename to modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/style.scss index f67439c0a64da..f4836148c0585 100644 --- a/modules/web-console/frontend/app/modules/branding/header-title.directive.js +++ b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/style.scss @@ -15,30 +15,30 @@ * limitations under the License. */ -const template = ` -

{{::title.text}}

-`; +web-console-header-content { + display: flex; + flex: 1; -/** - * @param {import('./branding.service').default} branding - */ -export default function factory(branding) { - function controller() { - const ctrl = this; - - ctrl.text = branding.headerText; + .web-console-header-content__right { + margin-left: auto; + display: flex; } - return { - restrict: 'E', - template, - controller, - controllerAs: 'title', - replace: true - }; + .web-console-header-item { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + padding: 0 20px; + border-left: 1px solid #dddddd; + height: 100%; + } } -factory.$inject = ['IgniteBranding']; - +.web-console-header-content__title { + display: flex; + align-items: center; + margin-left: 80px; + font-size: 1.4em; + margin-right: auto; +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/template.pug similarity index 73% rename from modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug rename to modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/template.pug index 0545db18b96a7..8f1d4f8747013 100644 --- a/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug +++ b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-content/template.pug @@ -13,3 +13,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +.web-console-header-content__right + .web-console-header-item(ng-if='$ctrl.showConnectedClusters') + connected-clusters + .web-console-header-item(ng-if='$ctrl.showDemoModeButton') + demo-mode-button + +.web-console-header-item(ng-if='$ctrl.showUserMenu') + user-menu \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-header/index.js b/modules/web-console/frontend/app/components/web-console-header/index.js index 3fc5c66579fc1..2a106d47f8129 100644 --- a/modules/web-console/frontend/app/components/web-console-header/index.js +++ b/modules/web-console/frontend/app/components/web-console-header/index.js @@ -17,9 +17,13 @@ import angular from 'angular'; import component from './component'; -import componentExtension from './components/web-console-header-extension/component'; +import userMenu from './components/user-menu/component'; +import {component as content} from './components/web-console-header-content/component'; +import {component as demo} from './components/demo-mode-button/component'; export default angular .module('ignite-console.web-console-header', []) .component('webConsoleHeader', component) - .component('webConsoleHeaderExtension', componentExtension); + .component('webConsoleHeaderContent', content) + .component('userMenu', userMenu) + .component('demoModeButton', demo); diff --git a/modules/web-console/frontend/app/components/web-console-header/style.scss b/modules/web-console/frontend/app/components/web-console-header/style.scss index 4edf6fd5e9318..e34328433dc7c 100644 --- a/modules/web-console/frontend/app/components/web-console-header/style.scss +++ b/modules/web-console/frontend/app/components/web-console-header/style.scss @@ -21,13 +21,24 @@ web-console-header { $nav-item-margin: 30px; $bottom-border-width: 4px; - display: block; - min-height: 85px; + display: flex; + flex-direction: row; + align-items: center; + height: 62px; font-family: Roboto; font-size: 16px; border-bottom: $bottom-border-width solid red; background: white; position: relative; + padding-left: 16px; + + &>ng-transclude { + display: flex; + flex-direction: row; + justify-content: flex-end; + height: 100%; + flex: 1; + } &:after { // Shows header shadow over absolutely positioned child content, @@ -39,6 +50,7 @@ web-console-header { position: absolute; overflow: hidden; bottom: -4px; + pointer-events: none; } .wch-slot { @@ -59,154 +71,26 @@ web-console-header { } } - .wch-content { - display: flex; - flex-direction: row; - flex-wrap: wrap; - align-items: center; - padding: 18px 30px; - position: relative; - height: 81px; - } - .wch-logo { height: 40px; } - .wch-nav-item, - .wch-nav-item > a { - cursor: pointer; - white-space: nowrap; - - &, &.active { - font-size: 16px; - font-weight: normal; - text-decoration: none; - } - - $color-default: $text-color; - $color-hover: $brand-primary; - $color-active: $brand-primary; - - &, a { - color: $color-default; - text-decoration: none; - } - - &, ul { - margin: 0; - list-style: none; - -webkit-padding-start: 0; - } - - &:hover { - color: $color-hover; - } - - &:focus, :focus, - &.active, .active { - color: $color-active; - text-decoration: none; - } - - &[disabled] { - opacity: 0.5; - cursor: default; - } - - .caret { - margin-left: 4px; - } - - &:not(:last-child) { - margin-right: $nav-item-margin; - } - - .dropdown-menu { - a { - &:hover, &:focus { - color: inherit; - } - } - li.active a { - color: inherit; - } - } - - .user-name-wrapper { - display: flex; - align-items: center; - - & > span { - max-width: 180px; - overflow: hidden; - text-overflow: ellipsis; - } - } - } - - .wch-demo-toggle { - margin-right: 10px; - - // When in demo mode and clusters are hidden - &:nth-last-child(2) { - margin-right: $nav-item-margin; - } - } - - .wch-notification { - line-height: 16px; - padding: 7px; - background: $brand-warning; - font-size: 16px; - text-align: center; - - a { - color: $brand-info; - } - - &+.wch-notification { - border-top: 1px solid darken($brand-warning, 15%); - } - } - - .wch-notification.wch-notification--demo { - z-index: 10000000; - position: fixed; - top: 0; - width: 100%; - - color: white; - background: $ignite-brand-success; - border: none; - - box-shadow: 0 0 15px #008eff; - animation: pulse 2s infinite; - - a { - color: white; - text-decoration: underline; - - &:hover { - color: #ffab40; - text-decoration: none; - } - } - } - .wch-additional-nav-items { display: flex; } -} -@keyframes pulse { - 0% { - box-shadow: 0 1px 2px #008eff; - } - 50% { - box-shadow: 0 1px 15px #008eff; + .web-console-header__togle-menu-button { + width: 40px; + height: 40px; + background: none !important; + border: none !important; + margin: 0 !important; + padding: 0 !important; + outline: none !important; + color: rgba(0, 0, 0, 0.54); + margin-right: 5px; } - 100% { - box-shadow: 0 1px 2px #008eff; + .web-console-header__togle-menu-button-hidden { + visibility: hidden; } } diff --git a/modules/web-console/frontend/app/components/web-console-header/template.pug b/modules/web-console/frontend/app/components/web-console-header/template.pug index 1d25b13289010..940f3a48f0bd3 100644 --- a/modules/web-console/frontend/app/components/web-console-header/template.pug +++ b/modules/web-console/frontend/app/components/web-console-header/template.pug @@ -14,23 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. -.wch-notification(ng-show='$ctrl.UserNotifications.isShown && $ctrl.UserNotifications.message' ng-bind-html='$ctrl.UserNotifications.message') - -.wch-notification(ng-show='$root.user.becomeUsed') - | You are currently viewing user #[strong {{$root.user.firstName}} {{$root.user.lastName}}] as administrator. #[a(ng-click='$root.revertIdentity()') Revert to your identity?] - -.wch-notification.wch-notification--demo(ng-if='$root.IgniteDemoMode') - div(ng-controller='demoController') - | You are now in #[b Demo Mode]. #[a(ng-click='closeDemo();') Close Demo?] - -.wch-content - connected-clusters(ng-if='$ctrl.$rootScope.user && !$ctrl.$rootScope.IgniteDemoMode && $ctrl.isConnectedClustersVisible && !$root.user.becomeUsed') - - a(ui-sref='default-state' ng-if='$ctrl.isAuthorized()').wch-logo-anchor - img.wch-logo(ng-src='{{::$ctrl.branding.headerLogo}}') - - a(ui-sref='landing' ng-if='!$ctrl.isAuthorized()').wch-logo-anchor - img.wch-logo(ng-src='{{::$ctrl.branding.headerLogo}}') - - .wch-slot.wch-slot-left(ng-transclude='slotLeft') - .wch-slot.wch-slot-right(ng-transclude='slotRight') +button.web-console-header__togle-menu-button( + type='button' + title='Toggle menu' + ng-click='$ctrl.toggleSidebar()' + ng-class='::{"web-console-header__togle-menu-button-hidden": $ctrl.hideMenuButton}' +) + svg(ignite-icon='menu') + +a(ui-sref='{{$ctrl.isAuthorized() ? "default-state" : "landing"}}').wch-logo-anchor + img.wch-logo(src='/images/ignite-logo.svg') + +ng-transclude diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/component.ts b/modules/web-console/frontend/app/components/web-console-sidebar/component.ts new file mode 100644 index 0000000000000..e86d3e512713a --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/component.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import template from './template.pug'; +import controller from './controller'; +import './style.scss'; + +export const component: ng.IComponentOptions = { + template, + controller +}; diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/controller.ts b/modules/web-console/frontend/app/components/web-console-sidebar/controller.ts new file mode 100644 index 0000000000000..bdf606b7bd481 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/controller.ts @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {AppStore, selectSidebarOpened} from '../../store'; + +export default class WebConsoleSidebar { + static $inject = ['$rootScope', 'Store'] + constructor( + private $rootScope: ng.IRootScopeService, + private store: AppStore + ) {} + sidebarOpened$ = this.store.state$.pipe(selectSidebarOpened()) + get showNavigation(): boolean { + return !!this.$rootScope.user; + } +} diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/index.ts b/modules/web-console/frontend/app/components/web-console-sidebar/index.ts new file mode 100644 index 0000000000000..717dca5a12711 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/index.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {component as overflow} from './web-console-sidebar-overflow/component'; +import {component as nav} from './web-console-sidebar-navigation/component'; +import {component as sidebar} from './component'; + +export default angular.module('sidebar', []) + .component('webConsoleSidebarOverflow', overflow) + .component('webConsoleSidebarNavigation', nav) + .component('webConsoleSidebar', sidebar); diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js b/modules/web-console/frontend/app/components/web-console-sidebar/style.scss similarity index 57% rename from modules/web-console/frontend/app/modules/branding/header-logo.directive.js rename to modules/web-console/frontend/app/components/web-console-sidebar/style.scss index 60e5d5d68aec0..b625333b8b23d 100644 --- a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js +++ b/modules/web-console/frontend/app/components/web-console-sidebar/style.scss @@ -15,25 +15,30 @@ * limitations under the License. */ -import template from './header-logo.pug'; +web-console-sidebar { + display: flex; + flex-direction: column; + background: white; + border-right: 1px #dddddd solid; + --width-narrow: 75px; + --width-wide: 280px; + --active-link-color: #ee2b27; + --inactive-link-color: #757575; + // Does not include notifications height + max-height: calc(100vh - var(--header-height)); + position: -webkit-sticky; + position: sticky; + top: var(--header-height); -/** - * @param {import('./branding.service').default} branding - */ -export default function factory(branding) { - function controller() { - const ctrl = this; - - ctrl.url = branding.headerLogo; + web-console-sidebar-overflow { + flex: 1; } - return { - restrict: 'E', - template, - controller, - controllerAs: 'logo', - replace: true - }; -} + web-console-footer { + flex: 0 0 auto; -factory.$inject = ['IgniteBranding']; + &:first-child { + margin-top: auto; + } + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/template.pug b/modules/web-console/frontend/app/components/web-console-sidebar/template.pug new file mode 100644 index 0000000000000..f02f2c408f873 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/template.pug @@ -0,0 +1,22 @@ +//- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +web-console-sidebar-overflow + web-console-sidebar-navigation(opened='$ctrl.sidebarOpened$|async:this' ng-if='$ctrl.showNavigation') +web-console-footer(ng-class=`{ + "web-console-footer__sidebar-opened": ($ctrl.sidebarOpened$|async:this), + "web-console-footer__sidebar-closed": !($ctrl.sidebarOpened$|async:this) +}`) \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/component.ts b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/component.ts new file mode 100644 index 0000000000000..4e4647f97605f --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/component.ts @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import './style.scss'; +import template from './template.pug'; +import controller from './controller'; + +export const component = { + controller, + template, + bindings: { + opened: '<' + } +}; diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/controller.ts b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/controller.ts new file mode 100644 index 0000000000000..8b5355a890782 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/controller.ts @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {AppStore, selectNavigationMenu} from '../../../store'; + +export default class WebConsoleSidebarNavigation { + static $inject = ['Store'] + constructor(private store: AppStore) {} + menu$ = this.store.state$.pipe(selectNavigationMenu()) +} diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss new file mode 100644 index 0000000000000..83c21a3956ea4 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +web-console-sidebar-navigation { + display: flex; + flex-direction: column; + padding-top: 14px; + + nav { + display: flex; + flex-direction: column; + list-style-type: none; + + &>li { + display: flex; + } + } + + .web-console-sidebar-navigation__link-icon { + width: 20px; + height: auto; + color: var(--inactive-link-color); + } + + .web-console-sidebar-navigation__link-narrow { + height: var(--width-narrow); + width: var(--width-narrow); + color: var(--inactive-link-color); + display: flex; + justify-content: center; + align-items: center; + + &:hover, &:focus { + color: var(--active-link-color); + + } + + &.web-console-sidebar-navigation__link-active { + color: var(--active-link-color); + position: relative; + + &:after { + content: ''; + display: block; + width: 2px; + // Line can't be outside of scrollable container + right: 0; + top: 0; + bottom: 0; + position: absolute; + background: var(--active-link-color); + } + .web-console-sidebar-navigation__link-icon { + color: inherit; + } + } + .web-console-sidebar-navigation__link-content { + display: none; + } + } + .web-console-sidebar-navigation__link-wide { + color: #393939; + text-decoration: none !important; + line-height: 19px; + font-size: 16px; + padding: 10px 28px 11px; + width: var(--width-wide); + display: inline-flex; + align-items: center; + + &:first-of-type { + margin-top: 18px; + } + + &:hover, &:focus { + color: var(--active-link-color); + + .web-console-sidebar-navigation__link-icon { + color: inherit; + } + } + &.web-console-sidebar-navigation__link-active { + color: white; + background: var(--active-link-color); + + .web-console-sidebar-navigation__link-icon { + color: inherit; + } + } + .web-console-sidebar-navigation__link-icon { + margin-right: 12px; + } + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/views/includes/header-right.pug b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/template.pug similarity index 51% rename from modules/web-console/frontend/views/includes/header-right.pug rename to modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/template.pug index d2d89aa616d57..c35de9dfdbf56 100644 --- a/modules/web-console/frontend/views/includes/header-right.pug +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/template.pug @@ -14,22 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. -.wch-demo-toggle(ng-if='!$root.user.becomeUsed' ng-controller='demoController') - button.btn-ignite.btn-ignite--success( - ng-if='!IgniteDemoMode' - ng-click='startDemo()' - ) Start Demo - -web-console-header-extension - -.wch-nav-item(ignite-userbar) - .user-name-wrapper( - ng-class='{active: $state.includes("base.settings")}' - ng-click='$event.stopPropagation()' - bs-dropdown='userbar.items' - data-placement='bottom-right' - data-trigger='hover focus' - data-container='self' +nav + li( + ng-repeat='item in $ctrl.menu$|async:this track by item.sref' + ng-if='!item.hidden' ) - span {{$root.user.firstName}} {{$root.user.lastName}} - span.caret + a.web-console-sidebar-navigation__link( + ui-sref='{{item.sref}}' + ui-sref-active=`{ + "web-console-sidebar-navigation__link-active": item.activeSref + }` + title='{{item.label|translate}}' + ng-class=`{ + 'web-console-sidebar-navigation__link-narrow': !$ctrl.opened, + 'web-console-sidebar-navigation__link-wide': $ctrl.opened + }` + tabindex='0' + ) + svg.web-console-sidebar-navigation__link-icon(ignite-icon='{{item.icon}}') + span.web-console-sidebar-navigation__link-content(translate='{{item.label}}') diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/component.ts b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/component.ts new file mode 100644 index 0000000000000..0f69e2dcabc6d --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/component.ts @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import controller from './controller'; +import template from './template.pug'; +import './style.scss'; + +export const component: ng.IComponentOptions = { + controller, + transclude: true, + template +}; diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/controller.ts b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/controller.ts new file mode 100644 index 0000000000000..d5403f4caa1f0 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/controller.ts @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ResizeObserver from 'resize-observer-polyfill'; + +export default class WebCOnsoleSidebarOverflow { + static $inject = ['$element', 'gridUtil', '$window'] + constructor(private el: JQLite, private gridUtil: {getScrollbarWidth(): number}, private $win: ng.IWindowService) {} + scrollEl!: JQLite + resizeObserver: ResizeObserver + $onInit() { + this.el.css('--scrollbar-width', this.gridUtil.getScrollbarWidth()); + } + $postLink() { + this.scrollEl[0].addEventListener('scroll', this.onScroll, {passive: true}); + this.resizeObserver = new ResizeObserver(() => this.applyStyles(this.scrollEl[0])); + this.resizeObserver.observe(this.el[0]); + } + $onDestroy() { + this.scrollEl[0].removeEventListener('scroll', this.onScroll); + this.resizeObserver.disconnect(); + } + applyStyles(target: HTMLElement) { + const {offsetHeight, scrollTop, scrollHeight} = target; + const top = scrollTop !== 0; + const bottom = Math.floor((offsetHeight + scrollTop)) !== Math.floor(scrollHeight); + + target.classList.toggle('top', top); + target.classList.toggle('bottom', bottom); + } + onScroll = (e: UIEvent) => { + this.$win.requestAnimationFrame(() => { + this.applyStyles(e.target as HTMLElement); + }); + } +} diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/style.scss b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/style.scss new file mode 100644 index 0000000000000..cbc6adca2a9b8 --- /dev/null +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/style.scss @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +web-console-sidebar-overflow { + overflow: hidden; + position: relative; + display: flex; + + &>.web-console-sidebar-overflow__scroll-root { + overflow-x: hidden; + overflow-y: scroll; + margin-right: calc(-1 * var(--scrollbar-width)); + + &.top:before { + display: block; + width: 100%; + content: ''; + height: 1px; + position: absolute; + top: -1px; + box-shadow: 0 0 9px 1px #a0a0a0; + transform: translateZ(1px); + } + &.bottom:after { + display: block; + width: 100%; + content: ''; + position: absolute; + bottom: 0; + box-shadow: 0 0 9px 1px #a0a0a0; + transform: translateZ(1px); + } + + &>ng-transclude { + display: block; + } + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/template.pug similarity index 82% rename from modules/web-console/frontend/app/modules/branding/powered-by-apache.pug rename to modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/template.pug index 6031235968c8a..5b70da601f414 100644 --- a/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-overflow/template.pug @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. -a(ng-if='poweredBy.show' href='//ignite.apache.org' target='_blank') - img(ng-src='/images/pb-ignite.png') +.web-console-sidebar-overflow__scroll-root(ng-ref='$ctrl.scrollEl' ng-ref-read='$element') + ng-transclude(ng-style='::{"--scrollbar-width": $ctrl.SCROLLBAR_WIDTH}') \ No newline at end of file diff --git a/modules/web-console/frontend/app/modules/branding/branding.module.js b/modules/web-console/frontend/app/modules/branding/branding.module.js index 20cd93e015f2b..8b23d04a8ef7a 100644 --- a/modules/web-console/frontend/app/modules/branding/branding.module.js +++ b/modules/web-console/frontend/app/modules/branding/branding.module.js @@ -19,12 +19,8 @@ import angular from 'angular'; import IgniteBranding from './branding.service'; -import igniteHeaderLogo from './header-logo.directive'; -import igniteHeaderTitle from './header-title.directive'; import igniteTerms from './terms.directive'; import igniteFeatures from './features.directive'; -import igniteFooter from './footer.directive'; -import ignitePoweredByApache from './powered-by-apache.directive'; angular .module('ignite-console.branding', [ @@ -41,9 +37,5 @@ angular tfMetaTagsProvider.setTitleSuffix(' – Apache Ignite Web Console'); }]) -.directive('ignitePoweredByApache', ignitePoweredByApache) -.directive('igniteHeaderLogo', igniteHeaderLogo) -.directive('igniteHeaderTitle', igniteHeaderTitle) .directive('igniteTerms', igniteTerms) -.directive('igniteFeatures', igniteFeatures) -.directive('igniteFooter', igniteFooter); +.directive('igniteFeatures', igniteFeatures); diff --git a/modules/web-console/frontend/app/modules/branding/branding.service.js b/modules/web-console/frontend/app/modules/branding/branding.service.js index 667dbfd0f6182..f184a4d2a0952 100644 --- a/modules/web-console/frontend/app/modules/branding/branding.service.js +++ b/modules/web-console/frontend/app/modules/branding/branding.service.js @@ -24,10 +24,6 @@ export default class { constructor(Version) { this.titleSuffix = ' - Apache Ignite Web Console'; - this.headerLogo = '/images/ignite-logo.svg'; - - this.headerText = 'Management console for Apache Ignite'; - this.showIgniteLogo = false; this.footerHtml = [ diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js index 6c14e4a4de518..9416bca6627ea 100644 --- a/modules/web-console/frontend/app/modules/demo/Demo.module.js +++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js @@ -71,34 +71,7 @@ function demoInterceptor(Demo) { demoInterceptor.$inject = ['Demo']; -/** - * @param {ng.IScope} $scope - * @param {import('@uirouter/angularjs').StateService} $state - * @param {ng.IWindowService} $window - * @param {ReturnType} Confirm - */ -function demoController($scope, $state, $window, Confirm) { - const _openTab = (stateName) => $window.open($state.href(stateName), '_blank'); - - $scope.startDemo = () => { - if (!$scope.user.demoCreated) - return _openTab('demo.reset'); - - Confirm.confirm('Would you like to continue with previous demo session?', true, false) - .then((resume) => { - if (resume) - return _openTab('demo.resume'); - - _openTab('demo.reset'); - }); - }; - - $scope.closeDemo = () => { - $window.close(); - }; -} -demoController.$inject = ['$scope', '$state', '$window', 'IgniteConfirm']; function igniteDemoInfoProvider() { const items = DEMO_INFO; @@ -211,6 +184,5 @@ angular .config(config) .provider('Demo', DemoProvider) .factory('demoInterceptor', demoInterceptor) -.controller('demoController', demoController) .provider('igniteDemoInfo', igniteDemoInfoProvider) .service('DemoInfo', DemoInfo); diff --git a/modules/web-console/frontend/app/modules/navbar/navbar.module.js b/modules/web-console/frontend/app/modules/navbar/navbar.module.js index 63588c7ac4509..e2a4f83bf0576 100644 --- a/modules/web-console/frontend/app/modules/navbar/navbar.module.js +++ b/modules/web-console/frontend/app/modules/navbar/navbar.module.js @@ -17,17 +17,10 @@ import angular from 'angular'; -import IgniteNavbar from './Navbar.provider'; import IgniteUserbar from './Userbar.provider'; -import igniteNavbar from './navbar.directive'; -import igniteUserbar from './userbar.directive'; - angular .module('ignite-console.navbar', [ ]) -.provider('IgniteNavbar', IgniteNavbar) -.provider('IgniteUserbar', IgniteUserbar) -.directive('igniteNavbar', igniteNavbar) -.directive('igniteUserbar', igniteUserbar); +.provider('IgniteUserbar', IgniteUserbar); diff --git a/modules/web-console/frontend/app/services/store.ts b/modules/web-console/frontend/app/services/store.ts new file mode 100644 index 0000000000000..19dde69e22ebd --- /dev/null +++ b/modules/web-console/frontend/app/services/store.ts @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {BehaviorSubject} from 'rxjs/BehaviorSubject'; +import {Subject} from 'rxjs/Subject'; +import {scan, tap} from 'rxjs/operators'; +import {merge} from 'rxjs/observable/merge'; + +interface Reducer { + (state: State, action: Actions): State +} + +export class Store { + static $inject = ['$injector']; + + actions$: Subject + state$: BehaviorSubject + private _reducers: Array> + + constructor(private $injector: ng.auto.IInjectorService) { + this.$injector = $injector; + + this.actions$ = new Subject(); + this.state$ = new BehaviorSubject({}); + this.actions$.pipe( + scan((state, action) => this._reducers.reduce((state, reducer) => reducer(state, action), state), void 0), + tap((state) => this.state$.next(state)) + ).subscribe(); + this._reducers = [(state) => state]; + } + + dispatch(action: Actions) { + this.actions$.next(action); + } + + addReducer(path: T, reducer: Reducer): void + addReducer(reducer: Reducer): void + addReducer(...args) { + if (typeof args[0] === 'string') { + const [path, reducer] = args; + this._reducers = [ + ...this._reducers, + (state = {}, action) => { + const pathState = reducer(state[path], action); + // Don't update root state if child state changes + return state[path] !== pathState ? {...state, [path]: pathState} : state; + } + ]; + } else { + const [reducer] = args; + this._reducers = [...this._reducers, reducer]; + } + } + + addEffects(EffectsClass) { + const instance = this.$injector.instantiate(EffectsClass); + return merge( + ...Object.keys(instance).filter((k) => k.endsWith('Effect$')).map((k) => instance[k]), + ).pipe(tap((a) => this.dispatch(a))).subscribe(); + } +} diff --git a/modules/web-console/frontend/app/store/actions/ui.ts b/modules/web-console/frontend/app/store/actions/ui.ts new file mode 100644 index 0000000000000..25d9ba048b509 --- /dev/null +++ b/modules/web-console/frontend/app/store/actions/ui.ts @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {NavigationMenuItem} from '../../types'; + +export const TOGGLE_SIDEBAR: 'TOGGLE_SIDEBAR' = 'TOGGLE_SIDEBAR'; +export const toggleSidebar = () => ({type: TOGGLE_SIDEBAR}); + +export const NAVIGATION_MENU_ITEM: 'NAVIGATION_MENU_ITEM' = 'NAVIGATION_MENU_ITEM'; +export const navigationMenuItem = (menuItem: NavigationMenuItem) => ({type: NAVIGATION_MENU_ITEM, menuItem}); + +export const HIDE_NAVIGATION_MENU_ITEM: 'HIDE_NAVIGATION_MENU_ITEM' = 'HIDE_NAVIGATION_MENU_ITEM'; +export const hideNavigationMenuItem = (label: NavigationMenuItem['label']) => ({type: HIDE_NAVIGATION_MENU_ITEM, label}); + +export const SHOW_NAVIGATION_MENU_ITEM: 'SHOW_NAVIGATION_MENU_ITEM' = 'SHOW_NAVIGATION_MENU_ITEM'; +export const showNavigationMenuItem = (label: NavigationMenuItem['label']) => ({type: SHOW_NAVIGATION_MENU_ITEM, label}); + +export type UIActions = + | ReturnType + | ReturnType + | ReturnType + | ReturnType; diff --git a/modules/web-console/frontend/app/modules/navbar/Navbar.provider.js b/modules/web-console/frontend/app/store/actions/user.ts similarity index 80% rename from modules/web-console/frontend/app/modules/navbar/Navbar.provider.js rename to modules/web-console/frontend/app/store/actions/user.ts index 9b905d7285aa8..34c8d2062674c 100644 --- a/modules/web-console/frontend/app/modules/navbar/Navbar.provider.js +++ b/modules/web-console/frontend/app/store/actions/user.ts @@ -15,16 +15,10 @@ * limitations under the License. */ -export default function() { - const items = []; +import {User} from '../../types'; - this.push = function(data) { - items.push(data); - }; +export const USER: 'USER' = 'USER'; +export const user = (user: User) => ({type: USER, user}); - this.$get = function() { - return items; - }; - - return this; -} +export type UserActions = + | ReturnType; diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js b/modules/web-console/frontend/app/store/effects/ui.ts similarity index 61% rename from modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js rename to modules/web-console/frontend/app/store/effects/ui.ts index 3f8b3deb5bfc9..2a635d6e362c6 100644 --- a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js +++ b/modules/web-console/frontend/app/store/effects/ui.ts @@ -15,25 +15,18 @@ * limitations under the License. */ -import template from './powered-by-apache.pug'; +import {AppStore, USER, ofType, hideNavigationMenuItem, showNavigationMenuItem} from '..'; +import {map} from 'rxjs/operators'; -/** - * @param {import('./branding.service').default} branding - */ -export default function factory(branding) { - function controller() { - const ctrl = this; - - ctrl.show = branding.showIgniteLogo; - } +export class UIEffects { + static $inject = ['Store'] + constructor(private store: AppStore) {} - return { - restrict: 'E', - template, - controller, - controllerAs: 'poweredBy' - }; + toggleQueriesNavItemEffect$ = this.store.actions$.pipe( + ofType(USER), + map((action) => { + const QUERY_LABEL = 'Queries'; + return action.user.becomeUsed ? hideNavigationMenuItem(QUERY_LABEL) : showNavigationMenuItem(QUERY_LABEL); + }) + ) } - -factory.$inject = ['IgniteBranding']; - diff --git a/modules/web-console/frontend/app/store/index.ts b/modules/web-console/frontend/app/store/index.ts new file mode 100644 index 0000000000000..fa4a878e1637d --- /dev/null +++ b/modules/web-console/frontend/app/store/index.ts @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Store} from '../services/store'; + +import {UIState, uiReducer} from './reducers/ui'; +import {UIActions} from './actions/ui'; +import {UIEffects} from './effects/ui'; + +import {UserActions} from './actions/user'; + +export * from './actions/ui'; +export * from './reducers/ui'; +export * from './selectors/ui'; + +export * from './actions/user'; + +export {ofType} from './ofType'; + +export type State = { + ui: UIState, +}; + +export type Actions = + | UserActions + | UIActions; + +export type AppStore = Store; + +register.$inject = ['Store']; +export function register(store: AppStore) { + store.addReducer('ui', uiReducer); + store.addEffects(UIEffects); +} diff --git a/modules/web-console/frontend/app/store/ofType.ts b/modules/web-console/frontend/app/store/ofType.ts new file mode 100644 index 0000000000000..3d3bd8e3d73e2 --- /dev/null +++ b/modules/web-console/frontend/app/store/ofType.ts @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {OperatorFunction} from 'rxjs/interfaces'; +import {filter} from 'rxjs/operators'; + +type Action = {type: string}; +export function ofType>(type: T): OperatorFunction +export function ofType(type): OperatorFunction { + return filter((action: U): boolean => type === action.type); +} + diff --git a/modules/web-console/frontend/app/store/reducers/ui.ts b/modules/web-console/frontend/app/store/reducers/ui.ts new file mode 100644 index 0000000000000..6f2859de6a74b --- /dev/null +++ b/modules/web-console/frontend/app/store/reducers/ui.ts @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {NavigationMenu} from '../../types'; +import {UIActions, TOGGLE_SIDEBAR, NAVIGATION_MENU_ITEM, HIDE_NAVIGATION_MENU_ITEM, SHOW_NAVIGATION_MENU_ITEM} from '..'; + +export type UIState = { + sidebarOpened: boolean, + navigationMenu: NavigationMenu +}; + +const defaults: UIState = { + sidebarOpened: false, + navigationMenu: [] +}; + +export function uiReducer(state: UIState = defaults, action: UIActions): UIState { + switch (action.type) { + case TOGGLE_SIDEBAR: + return {...state, sidebarOpened: !state.sidebarOpened}; + case NAVIGATION_MENU_ITEM: + return {...state, navigationMenu: [...state.navigationMenu, action.menuItem]}; + case HIDE_NAVIGATION_MENU_ITEM: + return { + ...state, + navigationMenu: state.navigationMenu.map((i) => i.label === action.label ? {...i, hidden: true} : i) + }; + case SHOW_NAVIGATION_MENU_ITEM: + return { + ...state, + navigationMenu: state.navigationMenu.map((i) => i.label === action.label ? {...i, hidden: false} : i) + }; + default: + return state; + } +} diff --git a/modules/web-console/frontend/app/store/reduxDebug.ts b/modules/web-console/frontend/app/store/reduxDebug.ts new file mode 100644 index 0000000000000..d7b1474ec052a --- /dev/null +++ b/modules/web-console/frontend/app/store/reduxDebug.ts @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {reducer, devTools} from '../components/page-configure/reduxDevtoolsIntegration'; +import {AppStore} from '.'; +import {filter, withLatestFrom, tap} from 'rxjs/operators'; + +run.$inject = ['Store']; + +export function run(store: AppStore) { + if (devTools) { + devTools.subscribe((e) => { + if (e.type === 'DISPATCH' && e.state) store.dispatch(e); + }); + + const ignoredActions = new Set([ + ]); + + store.actions$.pipe( + filter((e) => e.type !== 'DISPATCH'), + withLatestFrom(store.state$.skip(1)), + tap(([action, state]) => { + if (ignoredActions.has(action.type)) return; + devTools.send(action, state); + console.log(action); + }) + ).subscribe(); + + store.addReducer(reducer); + } +} diff --git a/modules/web-console/frontend/app/modules/branding/footer.directive.js b/modules/web-console/frontend/app/store/selectors/ui.ts similarity index 62% rename from modules/web-console/frontend/app/modules/branding/footer.directive.js rename to modules/web-console/frontend/app/store/selectors/ui.ts index 3c0bb83e740f5..0bd2fa92a95c5 100644 --- a/modules/web-console/frontend/app/modules/branding/footer.directive.js +++ b/modules/web-console/frontend/app/store/selectors/ui.ts @@ -15,25 +15,15 @@ * limitations under the License. */ -const template = ''; +import {State, AppStore} from '..'; +import {pluck, map} from 'rxjs/operators'; +import {pipe} from 'rxjs'; +import {memoize, orderBy} from 'lodash'; -/** - * @param {import('./branding.service').default} branding - */ -export default function factory(branding) { - function controller() { - const ctrl = this; - - ctrl.html = branding.footerHtml; - } - - return { - restrict: 'E', - template, - controller, - controllerAs: 'footer', - replace: true - }; -} +const orderMenu = (menu: Array) => orderBy(menu, 'order'); -factory.$inject = ['IgniteBranding']; +export const selectSidebarOpened = () => pluck('ui', 'sidebarOpened'); +export const selectNavigationMenu = () => pipe( + pluck('ui', 'navigationMenu'), + map(orderMenu) +); diff --git a/modules/web-console/frontend/app/types/index.ts b/modules/web-console/frontend/app/types/index.ts index df6a169d6a215..ad40c15308669 100644 --- a/modules/web-console/frontend/app/types/index.ts +++ b/modules/web-console/frontend/app/types/index.ts @@ -32,3 +32,33 @@ export interface IIgniteNg1StateDeclaration extends Ng1StateDeclaration { tfMetaTags: ITfMetatagsConfig, permission?: string } + +export type User = { + _id: string, + firstName: string, + lastName: string, + email: string, + phone?: string, + company: string, + country: string, + registered: string, + lastLogin: string, + lastActivity: string, + admin: boolean, + token: string, + resetPasswordToken: string, + // Assigned in UI + becomeUsed?: boolean +}; + +export type NavigationMenuItem = { + label: string, + icon: string, + order: number, + hidden?: boolean +} & ( + {sref: string, activeSref: string} | + {href: string} +); + +export type NavigationMenu = Array; diff --git a/modules/web-console/frontend/public/images/icons/cluster.icon.svg b/modules/web-console/frontend/public/images/icons/cluster.icon.svg new file mode 100644 index 0000000000000..757604970e52f --- /dev/null +++ b/modules/web-console/frontend/public/images/icons/cluster.icon.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/modules/web-console/frontend/public/images/icons/connectedClusters.icon.svg b/modules/web-console/frontend/public/images/icons/connectedClusters.icon.svg deleted file mode 100644 index f4053171c6591..0000000000000 --- a/modules/web-console/frontend/public/images/icons/connectedClusters.icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/modules/web-console/frontend/public/images/icons/downloadAgent.icon.svg b/modules/web-console/frontend/public/images/icons/downloadAgent.icon.svg new file mode 100644 index 0000000000000..afb51eeb3cf96 --- /dev/null +++ b/modules/web-console/frontend/public/images/icons/downloadAgent.icon.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/modules/web-console/frontend/public/images/icons/index.js b/modules/web-console/frontend/public/images/icons/index.js index 0951d32550bc8..2e4108545eabb 100644 --- a/modules/web-console/frontend/public/images/icons/index.js +++ b/modules/web-console/frontend/public/images/icons/index.js @@ -21,7 +21,6 @@ export {default as check} from './check.icon.svg'; export {default as checkmark} from './checkmark.icon.svg'; export {default as clock} from './clock.icon.svg'; export {default as collapse} from './collapse.icon.svg'; -export {default as connectedClusters} from './connectedClusters.icon.svg'; export {default as copy} from './copy.icon.svg'; export {default as cross} from './cross.icon.svg'; export {default as csv} from './csv.icon.svg'; @@ -43,3 +42,7 @@ export {default as refresh} from './refresh.icon.svg'; export {default as search} from './search.icon.svg'; export {default as sort} from './sort.icon.svg'; export {default as structure} from './structure.icon.svg'; +export {default as cluster} from './cluster.icon.svg'; +export {default as sql} from './sql.icon.svg'; +export {default as menu} from './menu.icon.svg'; +export {default as downloadAgent} from './downloadAgent.icon.svg'; diff --git a/modules/web-console/frontend/public/images/icons/menu.icon.svg b/modules/web-console/frontend/public/images/icons/menu.icon.svg new file mode 100644 index 0000000000000..392d42ec945b1 --- /dev/null +++ b/modules/web-console/frontend/public/images/icons/menu.icon.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/modules/web-console/frontend/public/images/icons/sql.icon.svg b/modules/web-console/frontend/public/images/icons/sql.icon.svg new file mode 100644 index 0000000000000..bd54eeed2a0b8 --- /dev/null +++ b/modules/web-console/frontend/public/images/icons/sql.icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss index ce14e45d6ec0a..3f601be4570bb 100644 --- a/modules/web-console/frontend/public/stylesheets/style.scss +++ b/modules/web-console/frontend/public/stylesheets/style.scss @@ -255,17 +255,70 @@ body { flex-direction: column; } -body > .wrapper, - // Nested ui-view, like on password reset state -body > .wrapper > ui-view { - display: flex; - flex-direction: column; +:root { + --page-side-padding: 30px; +} + +.wrapper { + --header-height: 62px; + min-height: 100vh; + display: grid; + grid-template-columns: auto 1fr; + grid-template-rows: auto var(--header-height) 1fr; + grid-template-areas: 'notifications notifications' 'header header' 'sidebar content'; + + permanent-notifications { + grid-area: notifications; + } + + web-console-header { + grid-area: header; + position: -webkit-sticky; + position: sticky; + top: 0; + z-index: 10; + transform: translateZ(1px); + } + + web-console-sidebar { + grid-area: sidebar; + transform: translateZ(1px); + } + + &>.content { + grid-area: content; + padding-left: var(--page-side-padding); + padding-right: var(--page-side-padding); + padding-bottom: var(--page-side-padding); + display: flex; + flex-direction: column; + + &>ui-view { + display: flex; + flex-direction: column; + flex: 1; + } + } + + &.wrapper-public { + grid-template-columns: 1fr; + grid-template-rows: auto var(--header-height) 1fr auto; + grid-template-areas: 'notifications' 'header' 'content' 'footer'; + } +} + +.public-page { + margin-left: auto; + margin-right: auto; flex: 1 0 auto; + width: 100%; + max-width: 530px; - .body-container { - flex: 1 0 auto; - padding-bottom: 30px; + .public-page__title { + font-size: 38px; + font-weight: 300; + margin: 30px 0 30px; } } @@ -374,7 +427,6 @@ i.btn { h1, h2, h3 { - user-select: none; font-weight: normal; /* Makes the vertical size of the text the same for all fonts. */ line-height: 1; @@ -2004,10 +2056,6 @@ html, body { position: absolute !important; } -.demo-mode { - padding-top: 30px; -} - // Fix for injecting svg icon into BS btn .btn--with-icon { display: flex; diff --git a/modules/web-console/frontend/views/base.pug b/modules/web-console/frontend/views/base.pug index 967d469b569e0..b14cdac9d7e0c 100644 --- a/modules/web-console/frontend/views/base.pug +++ b/modules/web-console/frontend/views/base.pug @@ -14,13 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. +permanent-notifications web-console-header - web-console-header-left - include ./includes/header-left - web-console-header-right - include ./includes/header-right + web-console-header-content -.container--responsive.body-container.flex-full-height - div(ui-view='').flex-full-height +web-console-sidebar -web-console-footer +ui-view.content \ No newline at end of file diff --git a/modules/web-console/frontend/views/includes/header-left.pug b/modules/web-console/frontend/views/includes/header-left.pug deleted file mode 100644 index 03e06167360c0..0000000000000 --- a/modules/web-console/frontend/views/includes/header-left.pug +++ /dev/null @@ -1,42 +0,0 @@ -//- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -a.wch-nav-item( - ui-sref='base.configuration.overview' - ui-sref-active='{active: "base.configuration"}' -) - | Configure - -.wch-nav-item(ng-if='!$root.user.becomeUsed') - a(ui-sref='base.sql.tabs.notebooks-list' ui-sref-active='active') - | Queries - -.wch-additional-nav-items(ignite-navbar) - .wch-nav-item(ng-repeat='item in navbar.items') - div(ng-if='$root.user.becomeUsed ? item.canBecomed : item.children' ng-click='$event.stopPropagation()' - ng-class='{active: $state.includes(item.sref)}' - bs-dropdown='item.children' - data-placement='bottom-left' - data-trigger='hover focus' - data-container='self' - aria-haspopup='true' - aria-expanded='false' - ) - span {{::item.text}} - span.caret - - a(ng-if='$root.user.becomeUsed ? item.canBecomed : !item.children' ui-sref='{{item.sref}}' ui-sref-active='active') - | {{::item.text}} diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.pug b/modules/web-console/frontend/views/public.pug similarity index 80% rename from modules/web-console/frontend/app/modules/branding/header-logo.pug rename to modules/web-console/frontend/views/public.pug index f0453d39da8ab..934ac56978a3f 100644 --- a/modules/web-console/frontend/app/modules/branding/header-logo.pug +++ b/modules/web-console/frontend/views/public.pug @@ -14,5 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. -a(ui-sref='signin') - img.navbar-brand(ng-src='{{logo.url}}') +web-console-header(hide-menu-button='true') + .web-console-header-content__title Management console for Apache Ignite +.content(ui-view='page') +web-console-footer.web-console-footer__page-bottom \ No newline at end of file From d2390bc7a4b6c0b8909c537b01111553ffa97104 Mon Sep 17 00:00:00 2001 From: Andrei Aleksandrov Date: Fri, 14 Dec 2018 19:48:53 +0300 Subject: [PATCH 251/403] IGNITE-10657 Thin clients should call onSessionExpired on disconnect to avoid memory leak - Fixes #5658. Signed-off-by: Alexey Goncharuk (cherry picked from commit 93ed0d5) --- .../odbc/ClientListenerAbstractConnectionContext.java | 6 ++++++ .../processors/odbc/jdbc/JdbcConnectionContext.java | 2 ++ .../processors/odbc/odbc/OdbcConnectionContext.java | 2 ++ .../processors/platform/client/ClientConnectionContext.java | 2 ++ 4 files changed, 12 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java index 856868de6ffc8..14fcf9d9f631d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerAbstractConnectionContext.java @@ -126,4 +126,10 @@ private AuthenticationContext authenticateExternal(String user, String pwd) thro return authCtx; } + + /** {@inheritDoc} */ + @Override public void onDisconnected() { + if (ctx.security().enabled()) + ctx.security().onSessionExpired(secCtx.subject().id()); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java index c80136d140c9b..8ba07903cbd9d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java @@ -207,5 +207,7 @@ public JdbcConnectionContext(GridKernalContext ctx, GridNioSession ses, GridSpin /** {@inheritDoc} */ @Override public void onDisconnected() { handler.onDisconnect(); + + super.onDisconnected(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java index 5592aabe0ce35..cf11270a200d3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java @@ -188,5 +188,7 @@ public OdbcConnectionContext(GridKernalContext ctx, GridNioSession ses, GridSpin /** {@inheritDoc} */ @Override public void onDisconnected() { handler.onDisconnect(); + + super.onDisconnected(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java index ffe38ca68870f..5e68c34ec56ef 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java @@ -143,6 +143,8 @@ public ClientResourceRegistry resources() { /** {@inheritDoc} */ @Override public void onDisconnected() { resReg.clean(); + + super.onDisconnected(); } /** From 919cc27a9938f73938bc99ec29d7e1774e694aa2 Mon Sep 17 00:00:00 2001 From: Andrei Aleksandrov Date: Mon, 17 Dec 2018 17:34:14 +0300 Subject: [PATCH 252/403] IGNITE-10716: ctx.security().onSessionExpired(subjid) is called on rest session expiration now. - Fixes #5682. Signed-off-by: Ilya Kasnacheev (cherry picked from commit 91a82a5) --- .../ignite/internal/processors/rest/GridRestProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java index 30d2f0a1328d2..7db07214bc526 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java @@ -500,6 +500,9 @@ public GridRestProcessor(GridKernalContext ctx) { if (ses.isTimedOut(sesTtl)) { clientId2SesId.remove(ses.clientId, ses.sesId); sesId2Ses.remove(ses.sesId, ses); + + if (ctx.security().enabled() && ses.secCtx != null && ses.secCtx.subject() != null) + ctx.security().onSessionExpired(ses.secCtx.subject().id()); } } } From e1e31743822d2c7858ea8cc42b1ad4dc8932e012 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Tue, 18 Dec 2018 10:19:02 +0700 Subject: [PATCH 253/403] IGNITE-10239 Fix wide nav menu top margin. (cherry picked from commit 0eb5bcc6989721d389a67be7880118313e02975b) --- .../web-console-sidebar-navigation/style.scss | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss index 83c21a3956ea4..b9c3e68a437f2 100644 --- a/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss +++ b/modules/web-console/frontend/app/components/web-console-sidebar/web-console-sidebar-navigation/style.scss @@ -72,6 +72,11 @@ web-console-sidebar-navigation { display: none; } } + + li:first-of-type > .web-console-sidebar-navigation__link-wide { + margin-top: 18px; + } + .web-console-sidebar-navigation__link-wide { color: #393939; text-decoration: none !important; @@ -82,10 +87,6 @@ web-console-sidebar-navigation { display: inline-flex; align-items: center; - &:first-of-type { - margin-top: 18px; - } - &:hover, &:focus { color: var(--active-link-color); From 26056ec27608253ba11f8f04533022183c735d14 Mon Sep 17 00:00:00 2001 From: vd-pyatkov Date: Thu, 6 Dec 2018 18:39:07 +0300 Subject: [PATCH 254/403] IGNITE-9980 Added additional flag to filter caches by types to control.sh - Fixes #5496. Signed-off-by: Alexey Goncharuk (cherry picked from commit 9199aac144844aa8e4a44e36e366af25dd90eb18) --- .../internal/commandline/CommandHandler.java | 24 +- .../commandline/cache/CacheArguments.java | 18 ++ .../verify/VerifyBackupPartitionsTaskV2.java | 97 ++++++- .../visor/verify/CacheFilterEnum.java | 52 ++++ .../verify/VisorIdleVerifyDumpTaskArg.java | 30 ++- .../ignite/util/GridCommandHandlerTest.java | 238 ++++++++++++++++-- 6 files changed, 425 insertions(+), 34 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 8d36769c92794..78fad291c44c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -101,6 +101,7 @@ import org.apache.ignite.internal.visor.tx.VisorTxTask; import org.apache.ignite.internal.visor.tx.VisorTxTaskArg; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue; import org.apache.ignite.internal.visor.verify.IndexValidationIssue; import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult; @@ -195,6 +196,9 @@ public class CommandHandler { /** */ private static final String CMD_SKIP_ZEROS = "--skipZeros"; + /** Cache filter. */ + private static final String CACHE_FILTER = "--cacheFilter"; + /** */ private static final String CMD_USER_ATTRIBUTES = "--user-attributes"; @@ -820,10 +824,14 @@ private void printCacheHelp() { nl(); log(i("Subcommands:")); - usageCache(LIST, "regexPattern", "[groups|seq]", "[nodeId]", op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE.text())); - usageCache(CONTENTION, "minQueueSize", "[nodeId]", "[maxPrint]"); - usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]"); - usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", "[nodeId]", op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K"))); + usageCache(LIST, "regexPattern", op(or("groups","seq")), op("nodeId"), op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE + .text())); + usageCache(CONTENTION, "minQueueSize", op("nodeId"), op("maxPrint")); + usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", + op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(), CacheFilterEnum.PERSISTENT.toString(), + CacheFilterEnum.NOT_PERSISTENT.toString()))); + usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", op("nodeId"), op(or(VI_CHECK_FIRST + " N", + VI_CHECK_THROUGH + " K"))); usageCache(DISTRIBUTION, or("nodeId", NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attName1,...,attrNameN")); usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN"); nl(); @@ -1143,7 +1151,7 @@ private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) th String path = executeTask( client, VisorIdleVerifyDumpTask.class, - new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros()) + new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros(), cacheArgs.getCacheFilterEnum()) ); log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'"); @@ -2070,6 +2078,12 @@ private CacheArguments parseAndValidateCacheArgs() { cacheArgs.dump(true); else if (CMD_SKIP_ZEROS.equals(nextArg)) cacheArgs.skipZeros(true); + else if (CACHE_FILTER.equals(nextArg)) { + String filter = nextArg("The cache filter should be specified. The following values can be " + + "used: " + Arrays.toString(CacheFilterEnum.values()) + '.').toUpperCase(); + + cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter)); + } else parseCacheNames(nextArg, cacheArgs); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java index f7146103a086d..a9ea6f76dfe1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java @@ -19,6 +19,7 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.internal.commandline.OutputFormat; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd; import org.jetbrains.annotations.Nullable; @@ -71,6 +72,23 @@ public class CacheArguments { /** Full config flag. */ private boolean fullConfig; + /** Cache filter. */ + private CacheFilterEnum cacheFilterEnum = CacheFilterEnum.ALL; + + /** + * @return Gets filter of caches, which will by checked. + */ + public CacheFilterEnum getCacheFilterEnum() { + return cacheFilterEnum; + } + + /** + * @param cacheFilterEnum Cache filter. + */ + public void setCacheFilterEnum(CacheFilterEnum cacheFilterEnum) { + this.cacheFilterEnum = cacheFilterEnum; + } + /** * @return Full config flag. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index acd8170c48b3e..5e872eb7c6121 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -37,21 +37,27 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobAdapter; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; +import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; +import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.resources.IgniteInstanceResource; @@ -205,7 +211,7 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) { for (String cacheName : arg.getCaches()) { DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName); - if (desc == null) { + if (desc == null || !isCacheMatchFilter(cacheName)) { missingCaches.add(cacheName); continue; @@ -214,15 +220,13 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) { grpIds.add(desc.groupId()); } - if (!missingCaches.isEmpty()) { - StringBuilder strBuilder = new StringBuilder("The following caches do not exist: "); - - for (String name : missingCaches) - strBuilder.append(name).append(", "); - - strBuilder.delete(strBuilder.length() - 2, strBuilder.length()); - - throw new IgniteException(strBuilder.toString()); + handlingMissedCaches(missingCaches); + } + else if (onlySpecificCaches()) { + for (DynamicCacheDescriptor desc : ignite.context().cache().cacheDescriptors().values()) { + if (desc.cacheConfiguration().getCacheMode() != CacheMode.LOCAL + && isCacheMatchFilter(desc.cacheName())) + grpIds.add(desc.groupId()); } } else { @@ -288,6 +292,79 @@ else if (e.getCause() instanceof IgniteException) return res; } + /** + * Checks and throw exception if caches was missed. + * + * @param missingCaches Missing caches. + */ + private void handlingMissedCaches(Set missingCaches) { + if (missingCaches.isEmpty()) + return; + + StringBuilder strBuilder = new StringBuilder("The following caches do not exist"); + + if (onlySpecificCaches()) { + VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; + + strBuilder.append(" or do not match to the given filter [") + .append(vdta.getCacheFilterEnum()) + .append("]: "); + } + else + strBuilder.append(": "); + + for (String name : missingCaches) + strBuilder.append(name).append(", "); + + strBuilder.delete(strBuilder.length() - 2, strBuilder.length()); + + throw new IgniteException(strBuilder.toString()); + } + + /** + * @return True if validates only specific caches, else false. + */ + private boolean onlySpecificCaches() { + if (arg instanceof VisorIdleVerifyDumpTaskArg) { + VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; + + return vdta.getCacheFilterEnum() != CacheFilterEnum.ALL; + } + + return false; + } + + /** + * @param cacheName Cache name. + */ + private boolean isCacheMatchFilter(String cacheName) { + if (arg instanceof VisorIdleVerifyDumpTaskArg) { + DataStorageConfiguration dsc = ignite.context().config().getDataStorageConfiguration(); + DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName); + CacheConfiguration cc = desc.cacheConfiguration(); + VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; + + switch (vdta.getCacheFilterEnum()) { + case SYSTEM: + return !desc.cacheType().userCache(); + + case NOT_PERSISTENT: + return desc.cacheType().userCache() && !GridCacheUtils.isPersistentCache(cc, dsc); + + case PERSISTENT: + return desc.cacheType().userCache() && GridCacheUtils.isPersistentCache(cc, dsc); + + case ALL: + break; + + default: + assert false: "Illegal cache filter: " + vdta.getCacheFilterEnum(); + } + } + + return true; + } + /** * @param grpCtx Group context. * @param part Local partition. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java new file mode 100644 index 0000000000000..4e87d5068eaa5 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.verify; + +import org.jetbrains.annotations.Nullable; + +/** + * Represents a type of cache(s) that can be used for comparing update counters and checksums between primary and backup partitions. + *
+ * @see org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2 + */ +public enum CacheFilterEnum { + /** All. */ + ALL, + + /** System. */ + SYSTEM, + + /** Persistent. */ + PERSISTENT, + + /** Not persistent. */ + NOT_PERSISTENT; + + /** Enumerated values. */ + private static final CacheFilterEnum[] VALS = values(); + + /** + * Efficiently gets enumerated value from its ordinal. + * + * @param ord Ordinal value. + * @return Enumerated value or {@code null} if ordinal out of range. + */ + public static @Nullable CacheFilterEnum fromOrdinal(int ord) { + return ord >= 0 && ord < VALS.length ? VALS[ord] : null; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java index 6316c2443a73d..29dfb5b7a45ba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java @@ -22,6 +22,7 @@ import java.io.ObjectOutput; import java.util.Set; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; /** * Arguments for {@link VisorIdleVerifyDumpTask}. @@ -29,9 +30,13 @@ public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg { /** */ private static final long serialVersionUID = 0L; + /** */ private boolean skipZeros; + /** Cache kind. */ + private CacheFilterEnum cacheFilterEnum; + /** * Default constructor. */ @@ -41,10 +46,12 @@ public VisorIdleVerifyDumpTaskArg() { /** * @param caches Caches. * @param skipZeros Skip zeros partitions. + * @param cacheFilterEnum Cache kind. */ - public VisorIdleVerifyDumpTaskArg(Set caches, boolean skipZeros) { + public VisorIdleVerifyDumpTaskArg(Set caches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) { super(caches); this.skipZeros = skipZeros; + this.cacheFilterEnum = cacheFilterEnum; } /** @@ -54,16 +61,37 @@ public boolean isSkipZeros() { return skipZeros; } + /** + * @return Kind fo cache. + */ + public CacheFilterEnum getCacheFilterEnum() { + return cacheFilterEnum; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { super.writeExternalData(out); + out.writeBoolean(skipZeros); + + U.writeEnum(out, cacheFilterEnum); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { super.readExternalData(protoVer, in); + skipZeros = in.readBoolean(); + + if (protoVer >= V2) + cacheFilterEnum = CacheFilterEnum.fromOrdinal(in.readByte()); + else + cacheFilterEnum = CacheFilterEnum.ALL; + } + + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V2; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 5448162ef588d..6540a031f0457 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -51,12 +51,15 @@ import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.AtomicConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.ConnectorConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.failure.FailureHandler; +import org.apache.ignite.failure.NoOpFailureHandler; import org.apache.ignite.internal.GridJobExecuteResponse; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; @@ -66,6 +69,7 @@ import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -81,15 +85,18 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.datastructures.GridCacheInternalKeyImpl; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.tx.VisorTxInfo; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -128,6 +135,12 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { /** Option is used for auto confirmation. */ private static final String CMD_AUTO_CONFIRMATION = "--yes"; + /** Atomic configuration. */ + private AtomicConfiguration atomicConfiguration; + + /** Additional data region configuration. */ + private DataRegionConfiguration dataRegionConfiguration; + /** * @return Folder in work directory. * @throws IgniteCheckedException If failed to resolve folder name. @@ -188,12 +201,19 @@ protected void injectTestSystemOut() { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + if (atomicConfiguration != null) + cfg.setAtomicConfiguration(atomicConfiguration); + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); cfg.setConnectorConfiguration(new ConnectorConfiguration()); - DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration( - new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024)); + DataStorageConfiguration memCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setMaxSize(50L * 1024 * 1024)); + + if (dataRegionConfiguration != null) + memCfg.setDataRegionConfigurations(dataRegionConfiguration); cfg.setDataStorageConfiguration(memCfg); @@ -208,6 +228,11 @@ protected void injectTestSystemOut() { return cfg; } + /** {@inheritDoc} */ + @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { + return new NoOpFailureHandler(); + } + /** * Test activation works via control.sh * @@ -889,7 +914,13 @@ public void testCacheIdleVerify() throws Exception { ignite.cluster().active(true); - createCacheAndPreload(ignite, 100); + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); + + for (int i = 0; i < 100; i++) + cache.put(i, i); injectTestSystemOut(); @@ -916,7 +947,15 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception { ignite.cluster().active(true); - createCacheAndPreload(ignite, 100); + int parts = 32; + + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); + + for (int i = 0; i < 100; i++) + cache.put(i, i); injectTestSystemOut(); @@ -928,7 +967,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception { corruptDataEntry(cacheCtx, 1, true, false); - corruptDataEntry(cacheCtx, 1 + cacheCtx.config().getAffinity().partitions() / 2, false, true); + corruptDataEntry(cacheCtx, 1 + parts / 2, false, true); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify")); @@ -945,11 +984,12 @@ public void testCacheIdleVerifyDump() throws Exception { ignite.cluster().active(true); - int keysCount = 20;//less than parts number for ability to check skipZeros flag. - - createCacheAndPreload(ignite, keysCount); + int parts = 32; - int parts = ignite.affinity(DEFAULT_CACHE_NAME).partitions(); + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); ignite.createCache(new CacheConfiguration<>() .setAffinity(new RendezvousAffinityFunction(false, parts)) @@ -958,6 +998,11 @@ public void testCacheIdleVerifyDump() throws Exception { injectTestSystemOut(); + int keysCount = 20;//less than parts number for ability to check skipZeros flag. + + for (int i = 0; i < keysCount; i++) + cache.put(i, i); + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME)); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME)); @@ -1025,26 +1070,33 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception { ignite.cluster().active(true); - createCacheAndPreload(ignite, 100); + int parts = 32; - injectTestSystemOut(); + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); + + for (int i = 0; i < 100; i++) + cache.put(i, i); - corruptingAndCheckDefaultCache(ignite); + corruptingAndCheckDefaultCache(ignite, parts, CacheFilterEnum.ALL); } /** * @param ignite Ignite. + * @param parts Parts. */ - private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException { + private void corruptingAndCheckDefaultCache(IgniteEx ignite, int parts, CacheFilterEnum cacheFilterEnum) throws IOException { injectTestSystemOut(); GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); corruptDataEntry(cacheCtx, 0, true, false); - corruptDataEntry(cacheCtx, cacheCtx.config().getAffinity().partitions() / 2, false, true); + corruptDataEntry(cacheCtx, parts / 2, false, true); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", cacheFilterEnum.toString())); Matcher fileNameMatcher = dumpFileNameMatcher(); @@ -1057,6 +1109,144 @@ private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException fail("Should be found dump with conflicts"); } + /** + * Tests that idle verify print partitions info over system caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnSystemCache() throws Exception { + int parts = 32; + + atomicConfiguration = new AtomicConfiguration() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + injectTestSystemOut(); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) { + ignite.semaphore("s" + i, i, false, true); + + ignite.atomicSequence("sq" + i, 0, true) + .incrementAndGet(); + } + + CacheGroupContext storedSysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-ds-group")); + + assertNotNull(storedSysCacheCtx); + + corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq0", + "default-ds-group"), true, false); + + corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq" + parts / 2, + "default-ds-group"), false, true); + + CacheGroupContext memorySysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-volatile-ds-group")); + + assertNotNull(memorySysCacheCtx); + + corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s0", + "default-volatile-ds-group"), true, false); + + corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s" + parts / 2, + "default-volatile-ds-group"), false, true); + + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum.SYSTEM.toString())); + + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("found 4 conflict partitions: [counterConflicts=2, " + + "hashConflicts=2]")); + } + else + fail("Should be found dump with conflicts"); + } + + /** + * Tests that idle verify print partitions info over persistence client caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnPersistenceClientCache() throws Exception { + int parts = 32; + + dataRegionConfiguration = new DataRegionConfiguration() + .setName("persistence-region") + .setMaxSize(100L * 1024 * 1024) + .setPersistenceEnabled(true); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2) + .setName(DEFAULT_CACHE_NAME) + .setDataRegionName("persistence-region")); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) + cache.put(i, i); + + corruptingAndCheckDefaultCache(ignite, parts, CacheFilterEnum.PERSISTENT); + } + + /** + * Tests that idle verify print partitions info over none-persistence client caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnNonePersistenceClientCache() throws Exception { + int parts = 32; + + dataRegionConfiguration = new DataRegionConfiguration() + .setName("none-persistence-region"); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2) + .setName(DEFAULT_CACHE_NAME) + .setDataRegionName("none-persistence-region")); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) + cache.put(i, i); + + injectTestSystemOut(); + + GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); + + corruptDataEntry(cacheCtx, 0, true, false); + + corruptDataEntry(cacheCtx, parts / 2, false, true); + + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum + .NOT_PERSISTENT.toString())); + + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("found 1 conflict partitions: [counterConflicts=0, " + + "hashConflicts=1]")); + } + else + fail("Should be found dump with conflicts"); + } + /** * Tests that idle verify print partitions info when node failing. * @@ -1479,7 +1669,13 @@ public void testCacheDistribution() throws Exception { ignite.cluster().active(true); - createCacheAndPreload(ignite, 100); + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); + + for (int i = 0; i < 100; i++) + cache.put(i, i); injectTestSystemOut(); @@ -1523,7 +1719,13 @@ public void testCacheResetLostPartitions() throws Exception { ignite.cluster().active(true); - createCacheAndPreload(ignite, 100); + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1) + .setName(DEFAULT_CACHE_NAME)); + + for (int i = 0; i < 100; i++) + cache.put(i, i); injectTestSystemOut(); @@ -1719,7 +1921,7 @@ private static class IncrementClosure implements EntryProcessor ctx, - int key, + Object key, boolean breakCntr, boolean breakData ) { From 4454a48bed306a255caf9b982f3fb7dff7301d87 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Tue, 18 Dec 2018 18:56:19 +0300 Subject: [PATCH 255/403] Revert "IGNITE-9980 Added additional flag to filter caches by types to control.sh - Fixes #5496." This reverts commit 26056ec27608253ba11f8f04533022183c735d14. --- .../internal/commandline/CommandHandler.java | 24 +- .../commandline/cache/CacheArguments.java | 18 -- .../verify/VerifyBackupPartitionsTaskV2.java | 97 +------ .../visor/verify/CacheFilterEnum.java | 52 ---- .../verify/VisorIdleVerifyDumpTaskArg.java | 30 +-- .../ignite/util/GridCommandHandlerTest.java | 238 ++---------------- 6 files changed, 34 insertions(+), 425 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 78fad291c44c0..8d36769c92794 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -101,7 +101,6 @@ import org.apache.ignite.internal.visor.tx.VisorTxTask; import org.apache.ignite.internal.visor.tx.VisorTxTaskArg; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; -import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue; import org.apache.ignite.internal.visor.verify.IndexValidationIssue; import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult; @@ -196,9 +195,6 @@ public class CommandHandler { /** */ private static final String CMD_SKIP_ZEROS = "--skipZeros"; - /** Cache filter. */ - private static final String CACHE_FILTER = "--cacheFilter"; - /** */ private static final String CMD_USER_ATTRIBUTES = "--user-attributes"; @@ -824,14 +820,10 @@ private void printCacheHelp() { nl(); log(i("Subcommands:")); - usageCache(LIST, "regexPattern", op(or("groups","seq")), op("nodeId"), op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE - .text())); - usageCache(CONTENTION, "minQueueSize", op("nodeId"), op("maxPrint")); - usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", - op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(), CacheFilterEnum.PERSISTENT.toString(), - CacheFilterEnum.NOT_PERSISTENT.toString()))); - usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", op("nodeId"), op(or(VI_CHECK_FIRST + " N", - VI_CHECK_THROUGH + " K"))); + usageCache(LIST, "regexPattern", "[groups|seq]", "[nodeId]", op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE.text())); + usageCache(CONTENTION, "minQueueSize", "[nodeId]", "[maxPrint]"); + usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]"); + usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", "[nodeId]", op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K"))); usageCache(DISTRIBUTION, or("nodeId", NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attName1,...,attrNameN")); usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN"); nl(); @@ -1151,7 +1143,7 @@ private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) th String path = executeTask( client, VisorIdleVerifyDumpTask.class, - new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros(), cacheArgs.getCacheFilterEnum()) + new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros()) ); log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'"); @@ -2078,12 +2070,6 @@ private CacheArguments parseAndValidateCacheArgs() { cacheArgs.dump(true); else if (CMD_SKIP_ZEROS.equals(nextArg)) cacheArgs.skipZeros(true); - else if (CACHE_FILTER.equals(nextArg)) { - String filter = nextArg("The cache filter should be specified. The following values can be " + - "used: " + Arrays.toString(CacheFilterEnum.values()) + '.').toUpperCase(); - - cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter)); - } else parseCacheNames(nextArg, cacheArgs); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java index a9ea6f76dfe1f..f7146103a086d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java @@ -19,7 +19,6 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.internal.commandline.OutputFormat; -import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd; import org.jetbrains.annotations.Nullable; @@ -72,23 +71,6 @@ public class CacheArguments { /** Full config flag. */ private boolean fullConfig; - /** Cache filter. */ - private CacheFilterEnum cacheFilterEnum = CacheFilterEnum.ALL; - - /** - * @return Gets filter of caches, which will by checked. - */ - public CacheFilterEnum getCacheFilterEnum() { - return cacheFilterEnum; - } - - /** - * @param cacheFilterEnum Cache filter. - */ - public void setCacheFilterEnum(CacheFilterEnum cacheFilterEnum) { - this.cacheFilterEnum = cacheFilterEnum; - } - /** * @return Full config flag. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index 5e872eb7c6121..acd8170c48b3e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -37,27 +37,21 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; -import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobAdapter; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; -import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.visor.verify.CacheFilterEnum; -import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.resources.IgniteInstanceResource; @@ -211,7 +205,7 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) { for (String cacheName : arg.getCaches()) { DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName); - if (desc == null || !isCacheMatchFilter(cacheName)) { + if (desc == null) { missingCaches.add(cacheName); continue; @@ -220,13 +214,15 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) { grpIds.add(desc.groupId()); } - handlingMissedCaches(missingCaches); - } - else if (onlySpecificCaches()) { - for (DynamicCacheDescriptor desc : ignite.context().cache().cacheDescriptors().values()) { - if (desc.cacheConfiguration().getCacheMode() != CacheMode.LOCAL - && isCacheMatchFilter(desc.cacheName())) - grpIds.add(desc.groupId()); + if (!missingCaches.isEmpty()) { + StringBuilder strBuilder = new StringBuilder("The following caches do not exist: "); + + for (String name : missingCaches) + strBuilder.append(name).append(", "); + + strBuilder.delete(strBuilder.length() - 2, strBuilder.length()); + + throw new IgniteException(strBuilder.toString()); } } else { @@ -292,79 +288,6 @@ else if (e.getCause() instanceof IgniteException) return res; } - /** - * Checks and throw exception if caches was missed. - * - * @param missingCaches Missing caches. - */ - private void handlingMissedCaches(Set missingCaches) { - if (missingCaches.isEmpty()) - return; - - StringBuilder strBuilder = new StringBuilder("The following caches do not exist"); - - if (onlySpecificCaches()) { - VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; - - strBuilder.append(" or do not match to the given filter [") - .append(vdta.getCacheFilterEnum()) - .append("]: "); - } - else - strBuilder.append(": "); - - for (String name : missingCaches) - strBuilder.append(name).append(", "); - - strBuilder.delete(strBuilder.length() - 2, strBuilder.length()); - - throw new IgniteException(strBuilder.toString()); - } - - /** - * @return True if validates only specific caches, else false. - */ - private boolean onlySpecificCaches() { - if (arg instanceof VisorIdleVerifyDumpTaskArg) { - VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; - - return vdta.getCacheFilterEnum() != CacheFilterEnum.ALL; - } - - return false; - } - - /** - * @param cacheName Cache name. - */ - private boolean isCacheMatchFilter(String cacheName) { - if (arg instanceof VisorIdleVerifyDumpTaskArg) { - DataStorageConfiguration dsc = ignite.context().config().getDataStorageConfiguration(); - DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName); - CacheConfiguration cc = desc.cacheConfiguration(); - VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; - - switch (vdta.getCacheFilterEnum()) { - case SYSTEM: - return !desc.cacheType().userCache(); - - case NOT_PERSISTENT: - return desc.cacheType().userCache() && !GridCacheUtils.isPersistentCache(cc, dsc); - - case PERSISTENT: - return desc.cacheType().userCache() && GridCacheUtils.isPersistentCache(cc, dsc); - - case ALL: - break; - - default: - assert false: "Illegal cache filter: " + vdta.getCacheFilterEnum(); - } - } - - return true; - } - /** * @param grpCtx Group context. * @param part Local partition. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java deleted file mode 100644 index 4e87d5068eaa5..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.visor.verify; - -import org.jetbrains.annotations.Nullable; - -/** - * Represents a type of cache(s) that can be used for comparing update counters and checksums between primary and backup partitions. - *
- * @see org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2 - */ -public enum CacheFilterEnum { - /** All. */ - ALL, - - /** System. */ - SYSTEM, - - /** Persistent. */ - PERSISTENT, - - /** Not persistent. */ - NOT_PERSISTENT; - - /** Enumerated values. */ - private static final CacheFilterEnum[] VALS = values(); - - /** - * Efficiently gets enumerated value from its ordinal. - * - * @param ord Ordinal value. - * @return Enumerated value or {@code null} if ordinal out of range. - */ - public static @Nullable CacheFilterEnum fromOrdinal(int ord) { - return ord >= 0 && ord < VALS.length ? VALS[ord] : null; - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java index 29dfb5b7a45ba..6316c2443a73d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java @@ -22,7 +22,6 @@ import java.io.ObjectOutput; import java.util.Set; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; /** * Arguments for {@link VisorIdleVerifyDumpTask}. @@ -30,13 +29,9 @@ public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg { /** */ private static final long serialVersionUID = 0L; - /** */ private boolean skipZeros; - /** Cache kind. */ - private CacheFilterEnum cacheFilterEnum; - /** * Default constructor. */ @@ -46,12 +41,10 @@ public VisorIdleVerifyDumpTaskArg() { /** * @param caches Caches. * @param skipZeros Skip zeros partitions. - * @param cacheFilterEnum Cache kind. */ - public VisorIdleVerifyDumpTaskArg(Set caches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) { + public VisorIdleVerifyDumpTaskArg(Set caches, boolean skipZeros) { super(caches); this.skipZeros = skipZeros; - this.cacheFilterEnum = cacheFilterEnum; } /** @@ -61,37 +54,16 @@ public boolean isSkipZeros() { return skipZeros; } - /** - * @return Kind fo cache. - */ - public CacheFilterEnum getCacheFilterEnum() { - return cacheFilterEnum; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { super.writeExternalData(out); - out.writeBoolean(skipZeros); - - U.writeEnum(out, cacheFilterEnum); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { super.readExternalData(protoVer, in); - skipZeros = in.readBoolean(); - - if (protoVer >= V2) - cacheFilterEnum = CacheFilterEnum.fromOrdinal(in.readByte()); - else - cacheFilterEnum = CacheFilterEnum.ALL; - } - - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return V2; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6540a031f0457..5448162ef588d 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -51,15 +51,12 @@ import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.AtomicConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.ConnectorConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; -import org.apache.ignite.failure.FailureHandler; -import org.apache.ignite.failure.NoOpFailureHandler; import org.apache.ignite.internal.GridJobExecuteResponse; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; @@ -69,7 +66,6 @@ import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; -import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -85,18 +81,15 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; -import org.apache.ignite.internal.processors.datastructures.GridCacheInternalKeyImpl; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.tx.VisorTxInfo; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; -import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -135,12 +128,6 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { /** Option is used for auto confirmation. */ private static final String CMD_AUTO_CONFIRMATION = "--yes"; - /** Atomic configuration. */ - private AtomicConfiguration atomicConfiguration; - - /** Additional data region configuration. */ - private DataRegionConfiguration dataRegionConfiguration; - /** * @return Folder in work directory. * @throws IgniteCheckedException If failed to resolve folder name. @@ -201,19 +188,12 @@ protected void injectTestSystemOut() { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - if (atomicConfiguration != null) - cfg.setAtomicConfiguration(atomicConfiguration); - cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); cfg.setConnectorConfiguration(new ConnectorConfiguration()); - DataStorageConfiguration memCfg = new DataStorageConfiguration() - .setDefaultDataRegionConfiguration(new DataRegionConfiguration() - .setMaxSize(50L * 1024 * 1024)); - - if (dataRegionConfiguration != null) - memCfg.setDataRegionConfigurations(dataRegionConfiguration); + DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024)); cfg.setDataStorageConfiguration(memCfg); @@ -228,11 +208,6 @@ protected void injectTestSystemOut() { return cfg; } - /** {@inheritDoc} */ - @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { - return new NoOpFailureHandler(); - } - /** * Test activation works via control.sh * @@ -914,13 +889,7 @@ public void testCacheIdleVerify() throws Exception { ignite.cluster().active(true); - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -947,15 +916,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception { ignite.cluster().active(true); - int parts = 32; - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -967,7 +928,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception { corruptDataEntry(cacheCtx, 1, true, false); - corruptDataEntry(cacheCtx, 1 + parts / 2, false, true); + corruptDataEntry(cacheCtx, 1 + cacheCtx.config().getAffinity().partitions() / 2, false, true); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify")); @@ -984,12 +945,11 @@ public void testCacheIdleVerifyDump() throws Exception { ignite.cluster().active(true); - int parts = 32; + int keysCount = 20;//less than parts number for ability to check skipZeros flag. - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); + createCacheAndPreload(ignite, keysCount); + + int parts = ignite.affinity(DEFAULT_CACHE_NAME).partitions(); ignite.createCache(new CacheConfiguration<>() .setAffinity(new RendezvousAffinityFunction(false, parts)) @@ -998,11 +958,6 @@ public void testCacheIdleVerifyDump() throws Exception { injectTestSystemOut(); - int keysCount = 20;//less than parts number for ability to check skipZeros flag. - - for (int i = 0; i < keysCount; i++) - cache.put(i, i); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME)); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME)); @@ -1070,33 +1025,26 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception { ignite.cluster().active(true); - int parts = 32; - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); + createCacheAndPreload(ignite, 100); - for (int i = 0; i < 100; i++) - cache.put(i, i); + injectTestSystemOut(); - corruptingAndCheckDefaultCache(ignite, parts, CacheFilterEnum.ALL); + corruptingAndCheckDefaultCache(ignite); } /** * @param ignite Ignite. - * @param parts Parts. */ - private void corruptingAndCheckDefaultCache(IgniteEx ignite, int parts, CacheFilterEnum cacheFilterEnum) throws IOException { + private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException { injectTestSystemOut(); GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); corruptDataEntry(cacheCtx, 0, true, false); - corruptDataEntry(cacheCtx, parts / 2, false, true); + corruptDataEntry(cacheCtx, cacheCtx.config().getAffinity().partitions() / 2, false, true); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", cacheFilterEnum.toString())); + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); Matcher fileNameMatcher = dumpFileNameMatcher(); @@ -1109,144 +1057,6 @@ private void corruptingAndCheckDefaultCache(IgniteEx ignite, int parts, CacheFil fail("Should be found dump with conflicts"); } - /** - * Tests that idle verify print partitions info over system caches. - * - * @throws Exception If failed. - */ - public void testCacheIdleVerifyDumpForCorruptedDataOnSystemCache() throws Exception { - int parts = 32; - - atomicConfiguration = new AtomicConfiguration() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(2); - - IgniteEx ignite = (IgniteEx)startGrids(3); - - ignite.cluster().active(true); - - injectTestSystemOut(); - - // Adding some assignments without deployments. - for (int i = 0; i < 100; i++) { - ignite.semaphore("s" + i, i, false, true); - - ignite.atomicSequence("sq" + i, 0, true) - .incrementAndGet(); - } - - CacheGroupContext storedSysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-ds-group")); - - assertNotNull(storedSysCacheCtx); - - corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq0", - "default-ds-group"), true, false); - - corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq" + parts / 2, - "default-ds-group"), false, true); - - CacheGroupContext memorySysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-volatile-ds-group")); - - assertNotNull(memorySysCacheCtx); - - corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s0", - "default-volatile-ds-group"), true, false); - - corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s" + parts / 2, - "default-volatile-ds-group"), false, true); - - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum.SYSTEM.toString())); - - Matcher fileNameMatcher = dumpFileNameMatcher(); - - if (fileNameMatcher.find()) { - String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); - - assertTrue(dumpWithConflicts.contains("found 4 conflict partitions: [counterConflicts=2, " + - "hashConflicts=2]")); - } - else - fail("Should be found dump with conflicts"); - } - - /** - * Tests that idle verify print partitions info over persistence client caches. - * - * @throws Exception If failed. - */ - public void testCacheIdleVerifyDumpForCorruptedDataOnPersistenceClientCache() throws Exception { - int parts = 32; - - dataRegionConfiguration = new DataRegionConfiguration() - .setName("persistence-region") - .setMaxSize(100L * 1024 * 1024) - .setPersistenceEnabled(true); - - IgniteEx ignite = (IgniteEx)startGrids(3); - - ignite.cluster().active(true); - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(2) - .setName(DEFAULT_CACHE_NAME) - .setDataRegionName("persistence-region")); - - // Adding some assignments without deployments. - for (int i = 0; i < 100; i++) - cache.put(i, i); - - corruptingAndCheckDefaultCache(ignite, parts, CacheFilterEnum.PERSISTENT); - } - - /** - * Tests that idle verify print partitions info over none-persistence client caches. - * - * @throws Exception If failed. - */ - public void testCacheIdleVerifyDumpForCorruptedDataOnNonePersistenceClientCache() throws Exception { - int parts = 32; - - dataRegionConfiguration = new DataRegionConfiguration() - .setName("none-persistence-region"); - - IgniteEx ignite = (IgniteEx)startGrids(3); - - ignite.cluster().active(true); - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(2) - .setName(DEFAULT_CACHE_NAME) - .setDataRegionName("none-persistence-region")); - - // Adding some assignments without deployments. - for (int i = 0; i < 100; i++) - cache.put(i, i); - - injectTestSystemOut(); - - GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); - - corruptDataEntry(cacheCtx, 0, true, false); - - corruptDataEntry(cacheCtx, parts / 2, false, true); - - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum - .NOT_PERSISTENT.toString())); - - Matcher fileNameMatcher = dumpFileNameMatcher(); - - if (fileNameMatcher.find()) { - String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); - - assertTrue(dumpWithConflicts.contains("found 1 conflict partitions: [counterConflicts=0, " + - "hashConflicts=1]")); - } - else - fail("Should be found dump with conflicts"); - } - /** * Tests that idle verify print partitions info when node failing. * @@ -1669,13 +1479,7 @@ public void testCacheDistribution() throws Exception { ignite.cluster().active(true); - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -1719,13 +1523,7 @@ public void testCacheResetLostPartitions() throws Exception { ignite.cluster().active(true); - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1) - .setName(DEFAULT_CACHE_NAME)); - - for (int i = 0; i < 100; i++) - cache.put(i, i); + createCacheAndPreload(ignite, 100); injectTestSystemOut(); @@ -1921,7 +1719,7 @@ private static class IncrementClosure implements EntryProcessor ctx, - Object key, + int key, boolean breakCntr, boolean breakData ) { From b0008008bbaf42f7bbfff380d2be96456c09413b Mon Sep 17 00:00:00 2001 From: vd-pyatkov Date: Thu, 6 Dec 2018 18:39:07 +0300 Subject: [PATCH 256/403] IGNITE-9980 Added additional flag to filter caches by types to control.sh - Fixes #5496. Signed-off-by: Alexey Goncharuk (cherry picked from commit 9199aac144844aa8e4a44e36e366af25dd90eb18) --- .../internal/commandline/CommandHandler.java | 24 ++- .../commandline/cache/CacheArguments.java | 18 ++ .../verify/VerifyBackupPartitionsTaskV2.java | 97 +++++++++-- .../visor/verify/CacheFilterEnum.java | 52 ++++++ .../verify/VisorIdleVerifyDumpTaskArg.java | 30 +++- .../ignite/util/GridCommandHandlerTest.java | 156 +++++++++++++++++- 6 files changed, 353 insertions(+), 24 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 8d36769c92794..78fad291c44c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -101,6 +101,7 @@ import org.apache.ignite.internal.visor.tx.VisorTxTask; import org.apache.ignite.internal.visor.tx.VisorTxTaskArg; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue; import org.apache.ignite.internal.visor.verify.IndexValidationIssue; import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult; @@ -195,6 +196,9 @@ public class CommandHandler { /** */ private static final String CMD_SKIP_ZEROS = "--skipZeros"; + /** Cache filter. */ + private static final String CACHE_FILTER = "--cacheFilter"; + /** */ private static final String CMD_USER_ATTRIBUTES = "--user-attributes"; @@ -820,10 +824,14 @@ private void printCacheHelp() { nl(); log(i("Subcommands:")); - usageCache(LIST, "regexPattern", "[groups|seq]", "[nodeId]", op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE.text())); - usageCache(CONTENTION, "minQueueSize", "[nodeId]", "[maxPrint]"); - usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]"); - usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", "[nodeId]", op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K"))); + usageCache(LIST, "regexPattern", op(or("groups","seq")), op("nodeId"), op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE + .text())); + usageCache(CONTENTION, "minQueueSize", op("nodeId"), op("maxPrint")); + usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", + op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(), CacheFilterEnum.PERSISTENT.toString(), + CacheFilterEnum.NOT_PERSISTENT.toString()))); + usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", op("nodeId"), op(or(VI_CHECK_FIRST + " N", + VI_CHECK_THROUGH + " K"))); usageCache(DISTRIBUTION, or("nodeId", NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attName1,...,attrNameN")); usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN"); nl(); @@ -1143,7 +1151,7 @@ private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) th String path = executeTask( client, VisorIdleVerifyDumpTask.class, - new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros()) + new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros(), cacheArgs.getCacheFilterEnum()) ); log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'"); @@ -2070,6 +2078,12 @@ private CacheArguments parseAndValidateCacheArgs() { cacheArgs.dump(true); else if (CMD_SKIP_ZEROS.equals(nextArg)) cacheArgs.skipZeros(true); + else if (CACHE_FILTER.equals(nextArg)) { + String filter = nextArg("The cache filter should be specified. The following values can be " + + "used: " + Arrays.toString(CacheFilterEnum.values()) + '.').toUpperCase(); + + cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter)); + } else parseCacheNames(nextArg, cacheArgs); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java index f7146103a086d..a9ea6f76dfe1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java @@ -19,6 +19,7 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.internal.commandline.OutputFormat; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd; import org.jetbrains.annotations.Nullable; @@ -71,6 +72,23 @@ public class CacheArguments { /** Full config flag. */ private boolean fullConfig; + /** Cache filter. */ + private CacheFilterEnum cacheFilterEnum = CacheFilterEnum.ALL; + + /** + * @return Gets filter of caches, which will by checked. + */ + public CacheFilterEnum getCacheFilterEnum() { + return cacheFilterEnum; + } + + /** + * @param cacheFilterEnum Cache filter. + */ + public void setCacheFilterEnum(CacheFilterEnum cacheFilterEnum) { + this.cacheFilterEnum = cacheFilterEnum; + } + /** * @return Full config flag. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index acd8170c48b3e..5e872eb7c6121 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -37,21 +37,27 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeJobAdapter; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; +import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; +import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.resources.IgniteInstanceResource; @@ -205,7 +211,7 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) { for (String cacheName : arg.getCaches()) { DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName); - if (desc == null) { + if (desc == null || !isCacheMatchFilter(cacheName)) { missingCaches.add(cacheName); continue; @@ -214,15 +220,13 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) { grpIds.add(desc.groupId()); } - if (!missingCaches.isEmpty()) { - StringBuilder strBuilder = new StringBuilder("The following caches do not exist: "); - - for (String name : missingCaches) - strBuilder.append(name).append(", "); - - strBuilder.delete(strBuilder.length() - 2, strBuilder.length()); - - throw new IgniteException(strBuilder.toString()); + handlingMissedCaches(missingCaches); + } + else if (onlySpecificCaches()) { + for (DynamicCacheDescriptor desc : ignite.context().cache().cacheDescriptors().values()) { + if (desc.cacheConfiguration().getCacheMode() != CacheMode.LOCAL + && isCacheMatchFilter(desc.cacheName())) + grpIds.add(desc.groupId()); } } else { @@ -288,6 +292,79 @@ else if (e.getCause() instanceof IgniteException) return res; } + /** + * Checks and throw exception if caches was missed. + * + * @param missingCaches Missing caches. + */ + private void handlingMissedCaches(Set missingCaches) { + if (missingCaches.isEmpty()) + return; + + StringBuilder strBuilder = new StringBuilder("The following caches do not exist"); + + if (onlySpecificCaches()) { + VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; + + strBuilder.append(" or do not match to the given filter [") + .append(vdta.getCacheFilterEnum()) + .append("]: "); + } + else + strBuilder.append(": "); + + for (String name : missingCaches) + strBuilder.append(name).append(", "); + + strBuilder.delete(strBuilder.length() - 2, strBuilder.length()); + + throw new IgniteException(strBuilder.toString()); + } + + /** + * @return True if validates only specific caches, else false. + */ + private boolean onlySpecificCaches() { + if (arg instanceof VisorIdleVerifyDumpTaskArg) { + VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; + + return vdta.getCacheFilterEnum() != CacheFilterEnum.ALL; + } + + return false; + } + + /** + * @param cacheName Cache name. + */ + private boolean isCacheMatchFilter(String cacheName) { + if (arg instanceof VisorIdleVerifyDumpTaskArg) { + DataStorageConfiguration dsc = ignite.context().config().getDataStorageConfiguration(); + DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName); + CacheConfiguration cc = desc.cacheConfiguration(); + VisorIdleVerifyDumpTaskArg vdta = (VisorIdleVerifyDumpTaskArg)arg; + + switch (vdta.getCacheFilterEnum()) { + case SYSTEM: + return !desc.cacheType().userCache(); + + case NOT_PERSISTENT: + return desc.cacheType().userCache() && !GridCacheUtils.isPersistentCache(cc, dsc); + + case PERSISTENT: + return desc.cacheType().userCache() && GridCacheUtils.isPersistentCache(cc, dsc); + + case ALL: + break; + + default: + assert false: "Illegal cache filter: " + vdta.getCacheFilterEnum(); + } + } + + return true; + } + /** * @param grpCtx Group context. * @param part Local partition. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java new file mode 100644 index 0000000000000..4e87d5068eaa5 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/CacheFilterEnum.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.verify; + +import org.jetbrains.annotations.Nullable; + +/** + * Represents a type of cache(s) that can be used for comparing update counters and checksums between primary and backup partitions. + *
+ * @see org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2 + */ +public enum CacheFilterEnum { + /** All. */ + ALL, + + /** System. */ + SYSTEM, + + /** Persistent. */ + PERSISTENT, + + /** Not persistent. */ + NOT_PERSISTENT; + + /** Enumerated values. */ + private static final CacheFilterEnum[] VALS = values(); + + /** + * Efficiently gets enumerated value from its ordinal. + * + * @param ord Ordinal value. + * @return Enumerated value or {@code null} if ordinal out of range. + */ + public static @Nullable CacheFilterEnum fromOrdinal(int ord) { + return ord >= 0 && ord < VALS.length ? VALS[ord] : null; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java index 6316c2443a73d..29dfb5b7a45ba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java @@ -22,6 +22,7 @@ import java.io.ObjectOutput; import java.util.Set; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; /** * Arguments for {@link VisorIdleVerifyDumpTask}. @@ -29,9 +30,13 @@ public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg { /** */ private static final long serialVersionUID = 0L; + /** */ private boolean skipZeros; + /** Cache kind. */ + private CacheFilterEnum cacheFilterEnum; + /** * Default constructor. */ @@ -41,10 +46,12 @@ public VisorIdleVerifyDumpTaskArg() { /** * @param caches Caches. * @param skipZeros Skip zeros partitions. + * @param cacheFilterEnum Cache kind. */ - public VisorIdleVerifyDumpTaskArg(Set caches, boolean skipZeros) { + public VisorIdleVerifyDumpTaskArg(Set caches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) { super(caches); this.skipZeros = skipZeros; + this.cacheFilterEnum = cacheFilterEnum; } /** @@ -54,16 +61,37 @@ public boolean isSkipZeros() { return skipZeros; } + /** + * @return Kind fo cache. + */ + public CacheFilterEnum getCacheFilterEnum() { + return cacheFilterEnum; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { super.writeExternalData(out); + out.writeBoolean(skipZeros); + + U.writeEnum(out, cacheFilterEnum); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { super.readExternalData(protoVer, in); + skipZeros = in.readBoolean(); + + if (protoVer >= V2) + cacheFilterEnum = CacheFilterEnum.fromOrdinal(in.readByte()); + else + cacheFilterEnum = CacheFilterEnum.ALL; + } + + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V2; } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 5448162ef588d..6bb999d00e235 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -51,6 +51,7 @@ import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.AtomicConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.ConnectorConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -66,6 +67,7 @@ import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -81,15 +83,18 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.datastructures.GridCacheInternalKeyImpl; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.tx.VisorTxInfo; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; +import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -128,6 +133,12 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest { /** Option is used for auto confirmation. */ private static final String CMD_AUTO_CONFIRMATION = "--yes"; + /** Atomic configuration. */ + private AtomicConfiguration atomicConfiguration; + + /** Additional data region configuration. */ + private DataRegionConfiguration dataRegionConfiguration; + /** * @return Folder in work directory. * @throws IgniteCheckedException If failed to resolve folder name. @@ -188,12 +199,19 @@ protected void injectTestSystemOut() { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + if (atomicConfiguration != null) + cfg.setAtomicConfiguration(atomicConfiguration); + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); cfg.setConnectorConfiguration(new ConnectorConfiguration()); - DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration( - new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024)); + DataStorageConfiguration memCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setMaxSize(50L * 1024 * 1024)); + + if (dataRegionConfiguration != null) + memCfg.setDataRegionConfigurations(dataRegionConfiguration); cfg.setDataStorageConfiguration(memCfg); @@ -1027,15 +1045,14 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception { createCacheAndPreload(ignite, 100); - injectTestSystemOut(); - - corruptingAndCheckDefaultCache(ignite); + corruptingAndCheckDefaultCache(ignite, CacheFilterEnum.ALL); } /** * @param ignite Ignite. + * @param cacheFilterEnum Filter enum. */ - private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException { + private void corruptingAndCheckDefaultCache(IgniteEx ignite, CacheFilterEnum cacheFilterEnum) throws IOException { injectTestSystemOut(); GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); @@ -1044,7 +1061,7 @@ private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException corruptDataEntry(cacheCtx, cacheCtx.config().getAffinity().partitions() / 2, false, true); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", cacheFilterEnum.toString())); Matcher fileNameMatcher = dumpFileNameMatcher(); @@ -1057,6 +1074,129 @@ private void corruptingAndCheckDefaultCache(IgniteEx ignite) throws IOException fail("Should be found dump with conflicts"); } + /** + * Tests that idle verify print partitions info over system caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnSystemCache() throws Exception { + int parts = 32; + + atomicConfiguration = new AtomicConfiguration() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + injectTestSystemOut(); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) { + ignite.semaphore("s" + i, i, false, true); + + ignite.atomicSequence("sq" + i, 0, true) + .incrementAndGet(); + } + + CacheGroupContext storedSysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-ds-group")); + + assertNotNull(storedSysCacheCtx); + + corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq0", + "default-ds-group"), true, false); + + corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq" + parts / 2, + "default-ds-group"), false, true); + + CacheGroupContext memorySysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-volatile-ds-group")); + + assertNotNull(memorySysCacheCtx); + + corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s0", + "default-volatile-ds-group"), true, false); + + corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s" + parts / 2, + "default-volatile-ds-group"), false, true); + + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum.SYSTEM.toString())); + + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("found 4 conflict partitions: [counterConflicts=2, " + + "hashConflicts=2]")); + } + else + fail("Should be found dump with conflicts"); + } + + /** + * Tests that idle verify print partitions info over persistence client caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnPersistenceClientCache() throws Exception { + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + createCacheAndPreload(ignite, 100); + + corruptingAndCheckDefaultCache(ignite, CacheFilterEnum.PERSISTENT); + } + + /** + * Tests that idle verify print partitions info over none-persistence client caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnNonePersistenceClientCache() throws Exception { + int parts = 32; + + dataRegionConfiguration = new DataRegionConfiguration() + .setName("none-persistence-region"); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2) + .setName(DEFAULT_CACHE_NAME) + .setDataRegionName("none-persistence-region")); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) + cache.put(i, i); + + injectTestSystemOut(); + + GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); + + corruptDataEntry(cacheCtx, 0, true, false); + + corruptDataEntry(cacheCtx, parts / 2, false, true); + + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum + .NOT_PERSISTENT.toString())); + + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("found 1 conflict partitions: [counterConflicts=0, " + + "hashConflicts=1]")); + } + else + fail("Should be found dump with conflicts"); + } + /** * Tests that idle verify print partitions info when node failing. * @@ -1719,7 +1859,7 @@ private static class IncrementClosure implements EntryProcessor ctx, - int key, + Object key, boolean breakCntr, boolean breakData ) { From 1e4955430da04b6674611c92acc0abf5a61a3aed Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Tue, 18 Dec 2018 17:48:48 +0300 Subject: [PATCH 257/403] IGNITE-10257 control.sh utility should request SSL keystore and truststore passwords if necessary - Fixes #5647. Signed-off-by: Alexey Goncharuk (cherry picked from commit ece5869bb9e7ff638352f07b63192ec2867afb8a) --- .../internal/commandline/CommandHandler.java | 208 +++++++++++------- 1 file changed, 129 insertions(+), 79 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 78fad291c44c0..450affa29b912 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -155,6 +155,7 @@ import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.CACHES; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.GROUPS; import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.SEQ; +import static org.apache.ignite.ssl.SslContextFactory.DFLT_SSL_PROTOCOL; /** * Class that execute several commands passed via command line. @@ -367,14 +368,7 @@ public class CommandHandler { private static final String UTILITY_NAME = "control.sh"; /** Common options. */ - private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"), - op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), - op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"), - op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"), - op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"), - op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"), - op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"), op(CMD_KEYSTORE, "KEYSTORE"), op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"), - op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"), op(CMD_TRUSTSTORE, "TRUSTSTORE"), op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD")); + private static final String COMMON_OPTIONS = String.join(" ", getCommonOptions()); /** Utility name with common options. */ private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS); @@ -400,6 +394,34 @@ public class CommandHandler { /** Check if experimental commands are enabled. Default {@code false}. */ private final boolean enableExperimental = IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false); + /** + * Creates list of common utility options. + * + * @return List of common utility options. + */ + private static List getCommonOptions() { + List list = new ArrayList<>(32); + + list.add(op(CMD_HOST, "HOST_OR_IP")); + list.add(op(CMD_PORT, "PORT")); + list.add(op(CMD_USER, "USER")); + list.add(op(CMD_PASSWORD, "PASSWORD")); + list.add(op(CMD_PING_INTERVAL, "PING_INTERVAL")); + list.add(op(CMD_PING_TIMEOUT, "PING_TIMEOUT")); + + list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]")); + list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]")); + list.add(op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM")); + list.add(op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE")); + list.add(op(CMD_KEYSTORE, "KEYSTORE")); + list.add(op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD")); + list.add(op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE")); + list.add(op(CMD_TRUSTSTORE, "TRUSTSTORE")); + list.add(op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD")); + + return list; + } + /** * Output specified string to console. * @@ -649,7 +671,9 @@ private R executeTask(GridClient client, Class> * @return List of hosts. */ private Stream> listHosts(GridClient client) throws GridClientException { - return client.compute().nodes(GridClientNode::connectable).stream() + return client.compute() + .nodes(GridClientNode::connectable) + .stream() .flatMap(node -> Stream.concat( node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(), node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream() @@ -661,14 +685,17 @@ private Stream> listHosts(GridClient clien * @param client Client. * @return List of hosts. */ - private Stream>> listHostsByClientNode(GridClient client) throws GridClientException { + private Stream>> listHostsByClientNode( + GridClient client + ) throws GridClientException { return client.compute().nodes(GridClientNode::connectable).stream() .map(node -> new IgniteBiTuple<>(node, Stream.concat( node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(), node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream() ) - .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList()))); + .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())) + ); } /** @@ -1836,7 +1863,7 @@ Arguments parseAndValidate(List rawArgs) { VisorTxTaskArg txArgs = null; - String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL; + String sslProtocol = DFLT_SSL_PROTOCOL; String sslCipherSuites = ""; @@ -2416,6 +2443,41 @@ private long nextLongArg(String lb) { } } + /** + * Requests password from console with message. + * + * @param msg Message. + * @return Password. + */ + private char[] requestPasswordFromConsole(String msg) { + Console console = System.console(); + + if (console == null) + throw new UnsupportedOperationException("Failed to securely read password (console is unavailable): " + msg); + else + return console.readPassword(msg); + } + + /** + * Requests user data from console with message. + * + * @param msg Message. + * @return Input user data. + */ + private String requestDataFromConsole(String msg) { + Console console = System.console(); + + if (console != null) + return console.readLine(msg); + else { + Scanner scanner = new Scanner(System.in); + + log(msg); + + return scanner.nextLine(); + } + } + /** * Check if raw arg is command or option. * @@ -2543,6 +2605,52 @@ private List split(String s, String delim) { .collect(Collectors.toList()); } + /** */ + private void printHelp() { + log("This utility can do the following commands:"); + + usage(i("Activate cluster:"), ACTIVATE); + usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION)); + usage(i("Print current cluster state:"), STATE); + usage(i("Print cluster baseline topology:"), BASELINE); + usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION)); + usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION)); + + if (enableExperimental) { + usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]"); + usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION)); + } + + log(i("View caches information in a cluster. For more details type:")); + log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2)); + nl(); + + log("By default commands affecting the cluster require interactive confirmation."); + log("Use " + CMD_AUTO_CONFIRMATION + " option to disable it."); + nl(); + + log("Default values:"); + log(i("HOST_OR_IP=" + DFLT_HOST, 2)); + log(i("PORT=" + DFLT_PORT, 2)); + log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2)); + log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2)); + log(i("SSL_PROTOCOL=" + DFLT_SSL_PROTOCOL, 2)); + log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2)); + log(i("KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + log(i("TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + nl(); + + log("Exit codes:"); + log(i(EXIT_CODE_OK + " - successful execution.", 2)); + log(i(EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.", 2)); + log(i(EXIT_CODE_CONNECTION_FAILED + " - connection failed.", 2)); + log(i(ERR_AUTHENTICATION_FAILED + " - authentication failed.", 2)); + log(i(EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.", 2)); + } + /** * Parse and execute command. * @@ -2557,48 +2665,7 @@ public int execute(List rawArgs) { try { if (F.isEmpty(rawArgs) || (rawArgs.size() == 1 && CMD_HELP.equalsIgnoreCase(rawArgs.get(0)))) { - log("This utility can do the following commands:"); - - usage(i("Activate cluster:"), ACTIVATE); - usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION)); - usage(i("Print current cluster state:"), STATE); - usage(i("Print cluster baseline topology:"), BASELINE); - usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); - usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); - usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); - usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION)); - usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION)); - - if (enableExperimental) { - usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]"); - usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION)); - } - - log(i("View caches information in a cluster. For more details type:")); - log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2)); - nl(); - - log("By default commands affecting the cluster require interactive confirmation."); - log("Use " + CMD_AUTO_CONFIRMATION + " option to disable it."); - nl(); - - log("Default values:"); - log(i("HOST_OR_IP=" + DFLT_HOST, 2)); - log(i("PORT=" + DFLT_PORT, 2)); - log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2)); - log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2)); - log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2)); - log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2)); - log(i("KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); - log(i("TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); - nl(); - - log("Exit codes:"); - log(i(EXIT_CODE_OK + " - successful execution.", 2)); - log(i(EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.", 2)); - log(i(EXIT_CODE_CONNECTION_FAILED + " - connection failed.", 2)); - log(i(ERR_AUTHENTICATION_FAILED + " - authentication failed.", 2)); - log(i(EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.", 2)); + printHelp(); return EXIT_CODE_OK; } @@ -2651,7 +2718,7 @@ public int execute(List rawArgs) { List sslProtocols = split(args.sslProtocol(), ","); - String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0); + String sslProtocol = F.isEmpty(sslProtocols) ? DFLT_SSL_PROTOCOL : sslProtocols.get(0); factory.setProtocol(sslProtocol); factory.setKeyAlgorithm(args.sslKeyAlgorithm()); @@ -2661,13 +2728,12 @@ public int execute(List rawArgs) { factory.setCipherSuites(split(args.getSslCipherSuites(), ",")); - if (args.sslKeyStorePath() == null) - throw new IllegalArgumentException("SSL key store location is not specified."); - factory.setKeyStoreFilePath(args.sslKeyStorePath()); if (args.sslKeyStorePassword() != null) factory.setKeyStorePassword(args.sslKeyStorePassword()); + else + factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: ")); factory.setKeyStoreType(args.sslKeyStoreType()); @@ -2678,6 +2744,8 @@ public int execute(List rawArgs) { if (args.sslTrustStorePassword() != null) factory.setTrustStorePassword(args.sslTrustStorePassword()); + else + factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: ")); factory.setTrustStoreType(args.sslTrustStoreType()); } @@ -2727,27 +2795,10 @@ public int execute(List rawArgs) { if (tryConnectMaxCount > 0 && isAuthError(e)) { log("Authentication error, try connection again."); - final Console console = System.console(); - - if (console != null) { - if (F.isEmpty(args.getUserName())) - args.setUserName(console.readLine("user: ")); - - args.setPassword(new String(console.readPassword("password: "))); - } - else { - Scanner scanner = new Scanner(System.in); - - if (F.isEmpty(args.getUserName())) { - log("user: "); - - args.setUserName(scanner.next()); - } + if (F.isEmpty(args.getUserName())) + args.setUserName(requestDataFromConsole("user: ")); - log("password: "); - - args.setPassword(scanner.next()); - } + args.setPassword(new String(requestPasswordFromConsole("password: "))); tryConnectAgain = true; @@ -2792,7 +2843,6 @@ public static void main(String[] args) { * * @return Last operation result; */ - @SuppressWarnings("unchecked") public T getLastOperationResult() { return (T)lastOperationRes; } From dbbe21bc62fe3ccacf83457cceb355876ea466d9 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Tue, 18 Dec 2018 17:34:27 +0700 Subject: [PATCH 258/403] IGNITE-9308 Add baseline topology command to REST API. - Fixes #5019. Signed-off-by: Andrey Novikov (cherry picked from commit 7c2ce4c7488253590885f75335b6d9209771fa46) --- .../client/suite/IgniteClientTestSuite.java | 2 + .../JettyRestProcessorBaselineSelfTest.java | 213 ++++++++++++++++++ .../processors/rest/GridRestCommand.java | 12 + .../processors/rest/GridRestProcessor.java | 6 + .../cluster/GridBaselineCommandHandler.java | 205 +++++++++++++++++ .../cluster/GridBaselineCommandResponse.java | 161 +++++++++++++ .../rest/request/GridRestBaselineRequest.java | 65 ++++++ .../http/jetty/GridJettyRestHandler.java | 16 ++ 8 files changed, 680 insertions(+) create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorBaselineSelfTest.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandHandler.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandResponse.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestBaselineRequest.java diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java index ffd7c81a39034..ca0f80b1e51d4 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java @@ -54,6 +54,7 @@ import org.apache.ignite.internal.processors.rest.ClientMemcachedProtocolSelfTest; import org.apache.ignite.internal.processors.rest.JettyRestProcessorAuthenticationWithCredsSelfTest; import org.apache.ignite.internal.processors.rest.JettyRestProcessorAuthenticationWithTokenSelfTest; +import org.apache.ignite.internal.processors.rest.JettyRestProcessorBaselineSelfTest; import org.apache.ignite.internal.processors.rest.JettyRestProcessorGetAllAsArrayTest; import org.apache.ignite.internal.processors.rest.JettyRestProcessorSignedSelfTest; import org.apache.ignite.internal.processors.rest.JettyRestProcessorUnsignedSelfTest; @@ -98,6 +99,7 @@ public static TestSuite suite() { suite.addTestSuite(JettyRestProcessorAuthenticationWithCredsSelfTest.class); suite.addTestSuite(JettyRestProcessorAuthenticationWithTokenSelfTest.class); suite.addTestSuite(JettyRestProcessorGetAllAsArrayTest.class); + suite.addTestSuite(JettyRestProcessorBaselineSelfTest.class); // Test TCP rest processor with original memcache client. suite.addTestSuite(ClientMemcachedProtocolSelfTest.class); diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorBaselineSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorBaselineSelfTest.java new file mode 100644 index 0000000000000..8f4548f8bce03 --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorBaselineSelfTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rest; + +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import org.apache.ignite.cluster.BaselineNode; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.rest.handlers.cluster.GridBaselineCommandResponse; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.configuration.WALMode.NONE; +import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_SUCCESS; + +/** + * Test REST with enabled authentication. + */ +public class JettyRestProcessorBaselineSelfTest extends JettyRestProcessorCommonSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true); + + super.beforeTestsStarted(); + + // We need to activate cluster. + grid(0).cluster().active(true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + grid(0).cluster().setBaselineTopology(grid(0).cluster().topologyVersion()); + } + + /** {@inheritDoc} */ + @Override protected String signature() { + return null; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setMaxSize(100 * 1024 * 1024) + .setPersistenceEnabled(true)) + .setWalMode(NONE); + + cfg.setDataStorageConfiguration(dsCfg); + + return cfg; + } + + /** + * @param nodes Collection of grid nodes. + * @return Collection of node consistent IDs for given collection of grid nodes. + */ + private static Collection nodeConsistentIds(@Nullable Collection nodes) { + if (nodes == null || nodes.isEmpty()) + return Collections.emptyList(); + + return F.viewReadOnly(nodes, n -> String.valueOf(n.consistentId())); + } + + /** + * @param content Content to check. + * @param baselineSz Expected baseline size. + * @param srvsSz Expected server nodes count. + */ + private void assertBaseline(String content, int baselineSz, int srvsSz) throws IOException { + assertNotNull(content); + assertFalse(content.isEmpty()); + + JsonNode node = JSON_MAPPER.readTree(content); + + assertEquals(STATUS_SUCCESS, node.get("successStatus").asInt()); + assertTrue(node.get("error").isNull()); + + assertNotSame(securityEnabled(), node.get("sessionToken").isNull()); + + JsonNode res = node.get("response"); + + assertFalse(res.isNull()); + + GridBaselineCommandResponse baseline = JSON_MAPPER.treeToValue(res, GridBaselineCommandResponse.class); + + assertTrue(baseline.isActive()); + assertEquals(grid(0).cluster().topologyVersion(), baseline.getTopologyVersion()); + assertEquals(baselineSz, baseline.getBaseline().size()); + assertEqualsCollections(nodeConsistentIds(grid(0).cluster().currentBaselineTopology()), baseline.getBaseline()); + assertEquals(srvsSz, baseline.getServers().size()); + assertEqualsCollections(nodeConsistentIds(grid(0).cluster().nodes()), baseline.getServers()); + } + + /** + * @throws Exception If failed. + */ + public void testBaseline() throws Exception { + int sz = gridCount(); + + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz); + + // Stop one node. It will stay in baseline. + stopGrid(sz - 1); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz - 1); + + // Start one node. Server node will be added, but baseline will not change. + startGrid(sz - 1); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz); + } + + /** + * @throws Exception If failed. + */ + public void testBaselineSet() throws Exception { + int sz = gridCount(); + + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz); + + startGrid(sz); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz + 1); + + assertBaseline(content(null, GridRestCommand.BASELINE_SET, "topVer", + String.valueOf(grid(0).cluster().topologyVersion())), sz + 1, sz + 1); + + stopGrid(sz); + + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz + 1, sz); + + assertBaseline(content(null, GridRestCommand.BASELINE_SET, "topVer", + String.valueOf(grid(0).cluster().topologyVersion())), sz, sz); + + startGrid(sz); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz + 1); + + ArrayList params = new ArrayList<>(); + int i = 1; + + for (BaselineNode n : grid(0).cluster().nodes()) { + params.add("consistentId" + i++); + params.add(String.valueOf(n.consistentId())); + } + + assertBaseline(content(null, GridRestCommand.BASELINE_SET, params.toArray(new String[0])), + sz + 1, sz + 1); + + stopGrid(sz); + } + + /** + * @throws Exception If failed. + */ + public void testBaselineAdd() throws Exception { + int sz = gridCount(); + + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz); + + startGrid(sz); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz + 1); + + assertBaseline(content(null, GridRestCommand.BASELINE_ADD, "consistentId1", + grid(sz).localNode().consistentId().toString()), sz + 1, sz + 1); + + stopGrid(sz); + } + + /** + * @throws Exception If failed. + */ + public void testBaselineRemove() throws Exception { + int sz = gridCount(); + + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz); + + startGrid(sz); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz, sz + 1); + + assertBaseline(content(null, GridRestCommand.BASELINE_SET, "topVer", + String.valueOf(grid(0).cluster().topologyVersion())), sz + 1, sz + 1); + + String consistentId = grid(sz).localNode().consistentId().toString(); + + stopGrid(sz); + assertBaseline(content(null, GridRestCommand.BASELINE_CURRENT_STATE), sz + 1, sz); + + assertBaseline(content(null, GridRestCommand.BASELINE_REMOVE, "consistentId1", + consistentId), sz, sz); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java index 587ed2ee8c1de..4046b1572a213 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java @@ -176,6 +176,18 @@ public enum GridRestCommand { /** */ CLUSTER_CURRENT_STATE("currentstate"), + /** */ + BASELINE_CURRENT_STATE("baseline"), + + /** */ + BASELINE_SET("setbaseline"), + + /** */ + BASELINE_ADD("addbaseline"), + + /** */ + BASELINE_REMOVE("removebaseline"), + /** */ AUTHENTICATE("authenticate"), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java index 7db07214bc526..f24c4d33a88ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java @@ -49,6 +49,7 @@ import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.auth.AuthenticationCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.cache.GridCacheCommandHandler; +import org.apache.ignite.internal.processors.rest.handlers.cluster.GridBaselineCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.cluster.GridChangeStateCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructuresCommandHandler; import org.apache.ignite.internal.processors.rest.handlers.log.GridLogCommandHandler; @@ -531,6 +532,7 @@ public GridRestProcessor(GridKernalContext ctx) { addHandler(new GridChangeStateCommandHandler(ctx)); addHandler(new AuthenticationCommandHandler(ctx)); addHandler(new UserActionCommandHandler(ctx)); + addHandler(new GridBaselineCommandHandler(ctx)); // Start protocols. startTcpProtocol(); @@ -895,6 +897,9 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit case CLUSTER_INACTIVE: case CLUSTER_ACTIVATE: case CLUSTER_DEACTIVATE: + case BASELINE_SET: + case BASELINE_ADD: + case BASELINE_REMOVE: perm = SecurityPermission.ADMIN_OPS; break; @@ -912,6 +917,7 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit case NAME: case LOG: case CLUSTER_CURRENT_STATE: + case BASELINE_CURRENT_STATE: case AUTHENTICATE: case ADD_USER: case REMOVE_USER: diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandHandler.java new file mode 100644 index 0000000000000..88fde87464647 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandHandler.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rest.handlers.cluster; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cluster.BaselineNode; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cluster.IgniteClusterEx; +import org.apache.ignite.internal.processors.rest.GridRestCommand; +import org.apache.ignite.internal.processors.rest.GridRestResponse; +import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandlerAdapter; +import org.apache.ignite.internal.processors.rest.request.GridRestBaselineRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestRequest; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.BASELINE_ADD; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.BASELINE_CURRENT_STATE; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.BASELINE_REMOVE; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.BASELINE_SET; + +/** + * + */ +public class GridBaselineCommandHandler extends GridRestCommandHandlerAdapter { + /** Supported commands. */ + private static final Collection SUPPORTED_COMMANDS = U.sealList(BASELINE_CURRENT_STATE, + BASELINE_SET, BASELINE_ADD, BASELINE_REMOVE); + + /** + * @param ctx Context. + */ + public GridBaselineCommandHandler(GridKernalContext ctx) { + super(ctx); + } + + /** {@inheritDoc} */ + @Override public Collection supportedCommands() { + return SUPPORTED_COMMANDS; + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture handleAsync(GridRestRequest req) { + assert req != null; + + assert SUPPORTED_COMMANDS.contains(req.command()); + assert req instanceof GridRestBaselineRequest : "Invalid type of baseline request."; + + + if (log.isDebugEnabled()) + log.debug("Handling baseline REST request: " + req); + + GridRestBaselineRequest req0 = (GridRestBaselineRequest)req; + + try { + IgniteClusterEx cluster = ctx.grid().cluster(); + + List consistentIds = req0.consistentIds(); + + switch (req0.command()) { + case BASELINE_CURRENT_STATE: { + // No-op. + + break; + } + + case BASELINE_SET: { + Long topVer = req0.topologyVersion(); + + if (topVer == null && consistentIds == null) + throw new IgniteCheckedException("Failed to handle request (either topVer or consistentIds should be specified)."); + + if (topVer != null) + cluster.setBaselineTopology(topVer); + else + cluster.setBaselineTopology(filterServerNodesByConsId(consistentIds)); + + break; + } + + case BASELINE_ADD: { + if (consistentIds == null) + throw new IgniteCheckedException(missingParameter("consistentIds")); + + Set baselineTop = new HashSet<>(currentBaseLine()); + + baselineTop.addAll(filterServerNodesByConsId(consistentIds)); + + cluster.setBaselineTopology(baselineTop); + + break; + } + + case BASELINE_REMOVE: { + if (consistentIds == null) + throw new IgniteCheckedException(missingParameter("consistentIds")); + + Collection baseline = currentBaseLine(); + + Set baselineTop = new HashSet<>(baseline); + + baselineTop.removeAll(filterNodesByConsId(baseline, consistentIds)); + + cluster.setBaselineTopology(baselineTop); + + break; + } + + default: + assert false : "Invalid command for baseline handler: " + req; + } + + return new GridFinishedFuture<>(new GridRestResponse(currentState())); + } + catch (IgniteCheckedException e) { + return new GridFinishedFuture<>(e); + } + finally { + if (log.isDebugEnabled()) + log.debug("Handled baseline REST request: " + req); + } + } + + /** + * @return Current baseline. + */ + private Collection currentBaseLine() { + Collection baselineNodes = ctx.grid().cluster().currentBaselineTopology(); + + return baselineNodes != null ? baselineNodes : Collections.emptyList(); + } + + /** + * Collect baseline topology command result. + * + * @return Baseline descriptor. + */ + private GridBaselineCommandResponse currentState() { + IgniteClusterEx cluster = ctx.grid().cluster(); + + Collection srvrs = cluster.forServers().nodes(); + + return new GridBaselineCommandResponse(cluster.active(), cluster.topologyVersion(), currentBaseLine(), srvrs); + } + + /** + * Filter passed nodes by consistent IDs. + * + * @param nodes Collection of nodes. + * @param consistentIds Collection of consistent IDs. + * @throws IllegalStateException In case of some consistent ID not found in nodes collection. + */ + private Collection filterNodesByConsId(Collection nodes, List consistentIds) { + Map nodeMap = + nodes.stream().collect(toMap(n -> n.consistentId().toString(), identity())); + + Collection filtered = new ArrayList<>(consistentIds.size()); + + for (Object consistentId : consistentIds) { + BaselineNode node = nodeMap.get(consistentId); + + if (node == null) + throw new IllegalStateException("Node not found for consistent ID: " + consistentId); + + filtered.add(node); + } + + return filtered; + } + + /** + * Filter server nodes by consistent IDs. + * + * @param consistentIds Collection of consistent IDs to add. + * @throws IllegalStateException In case of some consistent ID not found in nodes collection. + */ + private Collection filterServerNodesByConsId(List consistentIds) { + return filterNodesByConsId(ctx.grid().cluster().forServers().nodes(), consistentIds); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandResponse.java new file mode 100644 index 0000000000000..aae16c0ec1ad9 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridBaselineCommandResponse.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rest.handlers.cluster; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collection; +import org.apache.ignite.cluster.BaselineNode; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static java.util.stream.Collectors.toList; + +/** + * Result for baseline command. + */ +public class GridBaselineCommandResponse implements Externalizable { + /** */ + private static final long serialVersionUID = 0L; + + /** Cluster state. */ + private boolean active; + + /** Current topology version. */ + private long topVer; + + /** Current baseline nodes. */ + private Collection baseline; + + /** Current server nodes. */ + private Collection srvs; + + /** + * @param nodes Nodes to process. + * @return Collection of consistentIds. + */ + private static Collection consistentIds(Collection nodes) { + return nodes.stream().map(n -> String.valueOf(n.consistentId())).collect(toList()); + } + + /** + * Default constructor. + */ + public GridBaselineCommandResponse() { + // No-op. + } + + /** + * Constructor. + * + * @param active Cluster state. + * @param topVer Current topology version. + * @param baseline Current baseline nodes. + * @param srvs Current server nodes. + */ + GridBaselineCommandResponse( + boolean active, + long topVer, + Collection baseline, + Collection srvs + ) { + this.active = active; + this.topVer = topVer; + this.baseline = consistentIds(baseline); + this.srvs = consistentIds(srvs); + } + + /** + * @return Cluster state. + */ + public boolean isActive() { + return active; + } + + /** + * @param active Cluster active. + */ + public void setActive(boolean active) { + this.active = active; + } + + /** + * @return Current topology version. + */ + public long getTopologyVersion() { + return topVer; + } + + /** + * @param topVer Current topology version. + */ + public void setTopologyVersion(long topVer) { + this.topVer = topVer; + } + + /** + * @return Baseline nodes. + */ + public Collection getBaseline() { + return baseline; + } + + /** + * @param baseline Baseline nodes. + */ + public void setBaseline(Collection baseline) { + this.baseline = baseline; + } + + /** + * @return Server nodes. + */ + public Collection getServers() { + return srvs; + } + + /** + * @param srvs Server nodes. + */ + public void setServers(Collection srvs) { + this.srvs = srvs; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeBoolean(active); + out.writeLong(topVer); + U.writeCollection(out, baseline); + U.writeCollection(out, srvs); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + active = in.readBoolean(); + topVer = in.readLong(); + baseline = U.readCollection(in); + srvs = U.readCollection(in); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(GridBaselineCommandResponse.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestBaselineRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestBaselineRequest.java new file mode 100644 index 0000000000000..a3aa042e5d0db --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestBaselineRequest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rest.request; + +import java.util.List; +import org.apache.ignite.internal.util.typedef.internal.S; + +/** + * Grid command topology request. + */ +public class GridRestBaselineRequest extends GridRestRequest { + /** Topology version to set. */ + private Long topVer; + + /** Collection of consistent IDs to set. */ + private List consistentIds; + + /** + * @return Topology version to set. + */ + public Long topologyVersion() { + return topVer; + } + + /** + * @param topVer New topology version to set. + */ + public void topologyVersion(Long topVer) { + this.topVer = topVer; + } + + /** + * @return Collection of consistent IDs to set. + */ + public List consistentIds() { + return consistentIds; + } + + /** + * @param consistentIds New collection of consistent IDs to set. + */ + public void consistentIds(List consistentIds) { + this.consistentIds = consistentIds; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(GridRestBaselineRequest.class, this, super.toString()); + } +} diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java index c0a4fe1b5c30a..f502fac8a3a0f 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java +++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java @@ -51,6 +51,7 @@ import org.apache.ignite.internal.processors.rest.GridRestProtocolHandler; import org.apache.ignite.internal.processors.rest.GridRestResponse; import org.apache.ignite.internal.processors.rest.request.DataStructuresRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestBaselineRequest; import org.apache.ignite.internal.processors.rest.request.GridRestCacheRequest; import org.apache.ignite.internal.processors.rest.request.GridRestChangeStateRequest; import org.apache.ignite.internal.processors.rest.request.GridRestLogRequest; @@ -633,6 +634,7 @@ private Object convert(String type, Object obj) throws IgniteCheckedException { case CACHE_CAS: case CACHE_METRICS: case CACHE_SIZE: + case CACHE_UPDATE_TLL: case CACHE_METADATA: case CACHE_REPLACE: case CACHE_APPEND: @@ -759,6 +761,20 @@ else if (cmd == CLUSTER_ACTIVE || cmd == CLUSTER_ACTIVATE) break; } + case BASELINE_CURRENT_STATE: + case BASELINE_SET: + case BASELINE_ADD: + case BASELINE_REMOVE: { + GridRestBaselineRequest restReq0 = new GridRestBaselineRequest(); + + restReq0.topologyVersion(longValue("topVer", params, null)); + restReq0.consistentIds(values(null, "consistentId", params)); + + restReq = restReq0; + + break; + } + case AUTHENTICATE: { restReq = new GridRestRequest(); From 6717c1cd151d85855d5dce21466be309d1bc02f2 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 14 Dec 2018 12:11:18 +0300 Subject: [PATCH 259/403] IGNITE-10406 .NET Reset binary writer structures on reconnect This closes #5561 (cherry picked from commit 2a0f3dfafcac08459007abca25feaa874f99a9fb) --- .../Apache.Ignite.Core.Tests.csproj | 1 + .../CacheQueriesWithRestartServerTest.cs | 160 ++++++++++++++++++ .../Apache.Ignite.Core.Tests/ReconnectTest.cs | 70 +++++++- .../Impl/Binary/BinaryFullTypeDescriptor.cs | 11 ++ .../Impl/Binary/Marshaller.cs | 36 +++- .../Structure/BinaryStructureTracker.cs | 3 +- .../Common/CopyOnWriteConcurrentDictionary.cs | 18 +- .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 2 + 8 files changed, 281 insertions(+), 20 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesWithRestartServerTest.cs diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index e2d4146bee5e5..77b68baec172d 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -116,6 +116,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesWithRestartServerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesWithRestartServerTest.cs new file mode 100644 index 0000000000000..d54204a0baf18 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesWithRestartServerTest.cs @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Cache.Query +{ + using System.Linq; + using System.Threading; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Cache; + using Apache.Ignite.Core.Cache.Query; + using NUnit.Framework; + + /// + /// Tests queries behavior with client reconnect and server restart. + /// + public sealed class CacheQueriesRestartServerTest + { + /** */ + private IIgnite _client; + + /** */ + private IIgnite _server; + + /// + /// Sets up the fixture. + /// + [TestFixtureSetUp] + public void FixtureSetUp() + { + _server = StartGrid(0); + _client = StartGrid(0, true); + + TestUtils.WaitForCondition(() => _server.GetCluster().GetNodes().Count == 2, 1000); + } + + /// + /// Tears down the fixture. + /// + [TestFixtureTearDown] + public void FixtureTearDown() + { + Ignition.StopAll(true); + } + + /// + /// Tests that Scan query works after client reconnect with full cluster restart. + /// + [Test] + public void Test_ScanQueryAfterClientReconnect_ReturnsResults([Values(true, false)] bool emptyFilterObject) + { + var cache = _client.GetOrCreateCache("Test"); + cache.Put(1, new Item { Id = 20, Title = "test" }); + + Ignition.Stop(_server.Name, false); + _server = StartGrid(0); + WaitForReconnect(_client, 5000); + + cache = _client.GetOrCreateCache("Test"); + cache.Put(1, new Item { Id = 30, Title = "test" }); + + var filter = emptyFilterObject + ? (ICacheEntryFilter) new TestFilter() + : new TestFilterWithField {TestValue = 9}; + + var cursor = cache.Query(new ScanQuery(filter)); + var items = cursor.GetAll(); + + Assert.AreEqual(30, items.Single().Value.Id); + } + + /// + /// Starts the grid. + /// + private static IIgnite StartGrid(int i, bool client = false) + { + return Ignition.Start(new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + ClientMode = client, + IgniteInstanceName = client ? "client-" + i : "grid-" + i + }); + } + + /// + /// Waits for reconnect. + /// + private static void WaitForReconnect(IIgnite ignite, int timeout) + { + var evt = new ManualResetEventSlim(false); + + ignite.ClientReconnected += (sender, args) => evt.Set(); + + var restarted = evt.Wait(timeout); + Assert.IsTrue(restarted); + } + + /// + /// Test filter. + /// + private class TestFilter : ICacheEntryFilter + { + /** */ + public bool Invoke(ICacheEntry entry) + { + return entry.Value.Id > 10; + } + } + + /// + /// Test filter with field. + /// + private class TestFilterWithField : ICacheEntryFilter + { + /** */ + public int TestValue { get; set; } + + /** */ + public bool Invoke(ICacheEntry entry) + { + return entry.Value.Id > TestValue; + } + } + + private class Item : IBinarizable + { + /** */ + public int Id { get; set; } + + /** */ + public string Title { get; set; } + + /** */ + public void WriteBinary(IBinaryWriter writer) + { + writer.WriteInt("Id", Id); + writer.WriteString("Title", Title); + } + + /** */ + public void ReadBinary(IBinaryReader reader) + { + Id = reader.ReadInt("Id"); + Title = reader.ReadString("Title"); + } + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs index 5d40408c2bee7..e8aa60a5dfd4e 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs @@ -17,11 +17,14 @@ namespace Apache.Ignite.Core.Tests { + using System; using System.Threading; + using System.Threading.Tasks; using Apache.Ignite.Core.Cache; using Apache.Ignite.Core.Cache.Configuration; using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Lifecycle; + using Apache.Ignite.Core.Tests.Client.Cache; using Apache.Ignite.Core.Tests.Process; using NUnit.Framework; @@ -63,9 +66,9 @@ public void TestClusterRestart() client.ClientReconnected += (sender, args) => { eventArgs = args; }; - var cache = client.GetCache(CacheName); + var cache = client.GetCache(CacheName); - cache[1] = 1; + cache[1] = new Person(1); Ignition.Stop(server.Name, true); @@ -91,14 +94,14 @@ public void TestClusterRestart() Assert.IsTrue(eventArgs.HasClusterRestarted); // Refresh the cache instance and check that it works. - var cache1 = client.GetCache(CacheName); + var cache1 = client.GetCache(CacheName); Assert.AreEqual(0, cache1.GetSize()); - cache1[1] = 2; - Assert.AreEqual(2, cache1[1]); + cache1[1] = new Person(2); + Assert.AreEqual(2, cache1[1].Id); // Check that old cache instance still works. - Assert.AreEqual(2, cache.Get(1)); + Assert.AreEqual(2, cache.Get(1).Id); } /// @@ -163,6 +166,60 @@ public void TestFailedConnection() } } + /// + /// Tests writer structure cleanup after client reconnect with full cluster restart. + /// + [Test] + public void TestClusterRestart_ResetsCachedMetadataAndWriterStructures() + { + var serverCfg = new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + CacheConfiguration = new[] {new CacheConfiguration(CacheName)} + }; + + var clientCfg = new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + IgniteInstanceName = "client", + ClientMode = true + }; + + var server = Ignition.Start(serverCfg); + var client = Ignition.Start(clientCfg); + + Assert.AreEqual(2, client.GetCluster().GetNodes().Count); + + var evt = new ManualResetEventSlim(false); + client.ClientReconnected += (sender, args) => evt.Set(); + + var cache = client.GetCache(CacheName); + cache[1] = new Person(1); + + Task.Factory.StartNew(() => + { + while (!evt.IsSet) + { + try + { + cache[1] = new Person(1); + } + catch (Exception) + { + // Ignore exceptions while disconnected, keep on trying to populate writer structure cache. + } + } + }); + + Ignition.Stop(server.Name, true); + var server2 = Ignition.Start(serverCfg); + evt.Wait(); + + // Verify that we can deserialize on server (meta is resent properly). + cache[2] = new Person(2); + + var serverCache = server2.GetCache(CacheName); + Assert.AreEqual(2, serverCache[2].Id); + } + /// /// Starts the server process. /// @@ -173,7 +230,6 @@ private static IgniteProcess StartServerProcess(IgniteConfiguration cfg) "-J-DIGNITE_QUIET=false"); } - /// /// Test set up. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryFullTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryFullTypeDescriptor.cs index 50c8c275b4fc2..c4003448e90fa 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryFullTypeDescriptor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryFullTypeDescriptor.cs @@ -245,6 +245,17 @@ public void UpdateWriteStructure(int pathIdx, IList updat } } + /// + /// Resets writer structure. + /// + public void ResetWriteStructure() + { + lock (this) + { + _writerTypeStruct = null; + } + } + /** */ public void UpdateReadStructure(int pathIdx, IList updates) { diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Marshaller.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Marshaller.cs index 0a7b54ce9aeff..e1cc98fb8556f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Marshaller.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Marshaller.cs @@ -268,7 +268,7 @@ public BinaryReader StartUnmarshal(IBinaryStream stream, BinaryMode mode = Binar { return new BinaryReader(this, stream, mode, null); } - + /// /// Gets metadata for the given type ID. /// @@ -297,7 +297,7 @@ public void PutBinaryType(IBinaryTypeDescriptor desc) { Debug.Assert(desc != null); - GetBinaryTypeHandler(desc); // ensure that handler exists + GetBinaryTypeHandler(desc); // ensure that handler exists if (Ignite != null) { @@ -505,7 +505,7 @@ private BinaryFullTypeDescriptor AddUserType(Type type, int typeId, string typeN desc = desc == null ? new BinaryFullTypeDescriptor(type, typeId, typeName, true, _cfg.NameMapper, - _cfg.IdMapper, ser, false, AffinityKeyMappedAttribute.GetFieldNameFromAttribute(type), + _cfg.IdMapper, ser, false, AffinityKeyMappedAttribute.GetFieldNameFromAttribute(type), BinaryUtils.IsIgniteEnum(type), registered) : new BinaryFullTypeDescriptor(desc, type, ser, registered); @@ -576,8 +576,8 @@ private BinaryFullTypeDescriptor AddUserType(BinaryTypeConfiguration typeCfg, Ty // Type is found. var typeName = GetTypeName(type, nameMapper); int typeId = GetTypeId(typeName, idMapper); - var affKeyFld = typeCfg.AffinityKeyFieldName - ?? AffinityKeyMappedAttribute.GetFieldNameFromAttribute(type); + var affKeyFld = typeCfg.AffinityKeyFieldName + ?? AffinityKeyMappedAttribute.GetFieldNameFromAttribute(type); var serializer = GetSerializer(_cfg, typeCfg, type, typeId, nameMapper, idMapper, _log); return AddType(type, typeId, typeName, true, keepDeserialized, nameMapper, idMapper, serializer, @@ -655,7 +655,7 @@ private BinaryFullTypeDescriptor AddType(Type type, int typeId, string typeName, ThrowConflictingTypeError(typeName, conflictingType.TypeName, typeId); } - var descriptor = new BinaryFullTypeDescriptor(type, typeId, typeName, userType, nameMapper, idMapper, + var descriptor = new BinaryFullTypeDescriptor(type, typeId, typeName, userType, nameMapper, idMapper, serializer, keepDeserialized, affKeyFieldName, isEnum); if (RegistrationDisabled) @@ -783,6 +783,30 @@ public string GetTypeName(Type type, IBinaryNameMapper mapper = null) return GetTypeName(type.AssemblyQualifiedName, mapper); } + /// + /// Called when local client node has been reconnected to the cluster. + /// + /// Cluster restarted flag. + public void OnClientReconnected(bool clusterRestarted) + { + if (!clusterRestarted) + return; + + // Reset all binary structures. Metadata must be sent again. + // _idToDesc enumerator is thread-safe (returns a snapshot). + // If there are new descriptors added concurrently, they are fine (we are already connected). + + // Race is possible when serialization is started before reconnect (or even before disconnect) + // and finished after reconnect, meta won't be sent to cluster because it is assumed to be known, + // but operation will succeed. + // We don't support this use case. Users should handle reconnect events properly when cluster is restarted. + // Supporting this very rare use case will complicate the code a lot with little benefit. + foreach (var desc in _idToDesc) + { + desc.Value.ResetWriteStructure(); + } + } + /// /// Gets the name of the type. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Structure/BinaryStructureTracker.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Structure/BinaryStructureTracker.cs index ee2e7e17047df..bf3ea6a6ddab8 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Structure/BinaryStructureTracker.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Structure/BinaryStructureTracker.cs @@ -97,6 +97,7 @@ public void UpdateWriterStructure(BinaryWriter writer) { if (_curStructUpdates != null) { + // The following line assumes that cluster meta update will succeed (BinaryProcessor.PutBinaryTypes). _desc.UpdateWriteStructure(_curStructPath, _curStructUpdates); var marsh = writer.Marshaller; @@ -146,4 +147,4 @@ private int GetNewFieldId(string fieldName, byte fieldTypeId, int action) return fieldId; } } -} \ No newline at end of file +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/CopyOnWriteConcurrentDictionary.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/CopyOnWriteConcurrentDictionary.cs index cd26af7b74522..4d3d1f82150e9 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/CopyOnWriteConcurrentDictionary.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/CopyOnWriteConcurrentDictionary.cs @@ -19,6 +19,7 @@ namespace Apache.Ignite.Core.Impl.Common { using System; + using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; @@ -27,7 +28,8 @@ namespace Apache.Ignite.Core.Impl.Common /// Good for frequent reads / infrequent writes scenarios. /// [SuppressMessage("Microsoft.Naming", "CA1711:IdentifiersShouldNotHaveIncorrectSuffix")] - public class CopyOnWriteConcurrentDictionary + [SuppressMessage("Microsoft.Naming", "CA1710:IdentifiersShouldHaveCorrectSuffix")] + public class CopyOnWriteConcurrentDictionary : IEnumerable> { /** */ private volatile Dictionary _dict = new Dictionary(); @@ -88,13 +90,17 @@ public void Set(TKey key, TValue value) _dict = dict0; } } + + /** */ + public IEnumerator> GetEnumerator() + { + return _dict.GetEnumerator(); + } - /// - /// Determines whether the specified key exists in the dictionary. - /// - public bool ContainsKey(TKey key) + /** */ + IEnumerator IEnumerable.GetEnumerator() { - return _dict.ContainsKey(key); + return GetEnumerator(); } } } \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs index 42d9ed65e8889..ecec7684d3cf7 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs @@ -1002,6 +1002,8 @@ internal void OnClientDisconnected() /// Cluster restarted flag. internal void OnClientReconnected(bool clusterRestarted) { + _marsh.OnClientReconnected(clusterRestarted); + _clientReconnectTaskCompletionSource.TrySetResult(clusterRestarted); var handler = ClientReconnected; From 49f5bca160227d477d6df6424433515e224344a9 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Tue, 13 Nov 2018 16:59:56 +0300 Subject: [PATCH 260/403] IGNITE-10171: SQL: show original query in GridRunningQueryInfo. This closes #5349. (cherry picked from commit 8bf8b8efc2f26abe5bc6be95b928d738fe834c65) --- .../query/h2/sql/GridSqlQuerySplitter.java | 2 +- .../processors/query/RunningQueriesTest.java | 93 +++++++++++++++++++ .../IgniteBinaryCacheQueryTestSuite.java | 2 + 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 51bd84242f0a3..e575e4a6c567b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -202,7 +202,7 @@ public static GridCacheTwoStepQuery split( // subqueries because we do not have unique FROM aliases yet. GridSqlQuery qry = parse(prepared, false); - String originalSql = qry.getSQL(); + String originalSql = prepared.getSQL(); // debug("ORIGINAL", originalSql); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java new file mode 100644 index 0000000000000..ae30552e77f62 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/RunningQueriesTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query; + +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CyclicBarrier; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests for running queries. + */ +public class RunningQueriesTest extends GridCommonAbstractTest { + /** + * + */ + public void testQueriesOriginalText() throws Exception { + IgniteEx ignite = startGrid(0); + + IgniteCache cache = ignite.getOrCreateCache(new CacheConfiguration() + .setName("cache") + .setQueryEntities(Collections.singletonList(new QueryEntity(Integer.class, Integer.class))) + .setSqlFunctionClasses(TestSQLFunctions.class) + ); + + cache.put(0, 0); + + GridTestUtils.runAsync(() -> cache.query(new SqlFieldsQuery( + "SELECT * FROM /* comment */ Integer WHERE awaitBarrier() = 0")).getAll()); + + GridTestUtils.runAsync(() -> cache.query(new SqlQuery(Integer.class, + "FROM /* comment */ Integer WHERE awaitBarrier() = 0")).getAll()); + + TestSQLFunctions.barrier.await(); + + Collection runningQueries = ignite.context().query().runningQueries(-1); + + TestSQLFunctions.barrier.await(); + + assertEquals(2, runningQueries.size()); + + for (GridRunningQueryInfo info : runningQueries) + assertTrue("Failed to find comment in query: " + info.query(), info.query().contains("/* comment */")); + } + + /** + * Utility class with custom SQL functions. + */ + public static class TestSQLFunctions { + /** Barrier. */ + static CyclicBarrier barrier = new CyclicBarrier(3); + + /** + * Await cyclic barrier twice, first time to wait for enter method, second time to wait for collecting running + * queries. + */ + @QuerySqlFunction + public static long awaitBarrier() { + try { + barrier.await(); + barrier.await(); + } + catch (Exception ignored) { + // No-op. + } + + return 0; + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java index 8a60c7d72ac70..f8ee758cf7c40 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java @@ -24,6 +24,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanSelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanWithEventsSelfTest; import org.apache.ignite.internal.processors.cache.BigEntryQueryTest; +import org.apache.ignite.internal.processors.query.RunningQueriesTest; /** * Cache query suite with binary marshaller. @@ -42,6 +43,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheBinaryObjectsScanSelfTest.class); suite.addTestSuite(IgniteCacheBinaryObjectsScanWithEventsSelfTest.class); suite.addTestSuite(BigEntryQueryTest.class); + suite.addTestSuite(RunningQueriesTest.class); suite.addTestSuite(BinaryMetadataConcurrentUpdateWithIndexesTest.class); From 6322ba19d3410092bb37431cb38a15042db8e8a6 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 20 Dec 2018 12:00:38 +0300 Subject: [PATCH 261/403] IGNITE-10465 Wait for local join before start TTL cleanup worker - Fixes #5705. Signed-off-by: Pavel Kovalenko (cherry picked from commit 29588c5) Signed-off-by: Pavel Kovalenko --- .../GridCacheSharedTtlCleanupManager.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java index ad2342b15110f..3934ad94a0867 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java @@ -130,20 +130,21 @@ private class CleanupWorker extends GridWorker { Throwable err = null; try { + cctx.discovery().localJoin(); + + assert !cctx.kernalContext().recoveryMode(); + while (!isCancelled()) { boolean expiredRemains = false; - // TTL cleanup is allowed only when node joined to topology. - if (!cctx.kernalContext().recoveryMode()) { - for (GridCacheTtlManager mgr : mgrs) { - updateHeartbeat(); + for (GridCacheTtlManager mgr : mgrs) { + updateHeartbeat(); - if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT)) - expiredRemains = true; + if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT)) + expiredRemains = true; - if (isCancelled()) - return; - } + if (isCancelled()) + return; } updateHeartbeat(); From 9747363d766e1d8e2714f3907200af5e96049fc6 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 19 Dec 2018 22:49:22 +0300 Subject: [PATCH 262/403] ignite-9213 Do not use IgniteInterruptedCheckedException to complete futures on node stop --- .../GridCachePartitionExchangeManager.java | 2 +- .../GridDhtPartitionsExchangeFuture.java | 4 +- .../CacheLockReleaseNodeLeaveTest.java | 107 +++++++++++++++++- 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 5a93b5acdb3e5..453b228b02379 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -763,7 +763,7 @@ public static Object rebalanceTopic(int idx) { stopErr = cctx.kernalContext().clientDisconnected() ? new IgniteClientDisconnectedCheckedException(cctx.kernalContext().cluster().clientReconnectFuture(), "Client node disconnected: " + cctx.igniteInstanceName()) : - new IgniteInterruptedCheckedException("Node is stopping: " + cctx.igniteInstanceName()); + new IgniteCheckedException("Node is stopping: " + cctx.igniteInstanceName()); // Stop exchange worker U.cancel(exchWorker); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 1f6c9679d75fe..077625678e8c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -845,7 +845,9 @@ else if (msg instanceof WalStateAbstractMessage) exchLog.info("Finished exchange init [topVer=" + topVer + ", crd=" + crdNode + ']'); } catch (IgniteInterruptedCheckedException e) { - onDone(e); + assert cctx.kernalContext().isStopping(); + + onDone(new IgniteCheckedException("Node stopped")); throw e; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java index 844a4300dadab..e4f19f3fdc7b3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java @@ -20,6 +20,7 @@ import java.util.ArrayDeque; import java.util.Queue; import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import org.apache.ignite.Ignite; @@ -140,8 +141,6 @@ public void testLockRelease() throws Exception { * @throws Exception If failed. */ public void testLockTopologyChange() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-9213"); - final int nodeCnt = 5; int threadCnt = 8; final int keys = 100; @@ -163,9 +162,86 @@ public void testLockTopologyChange() throws Exception { Lock lock = cache.lock(i); lock.lock(); - cache.put(i, i); + try { + cache.put(i, i); + } + finally { + lock.unlock(); + } + } + } + } + }, threadCnt, "test-lock-thread"); + + q.add(f); + + U.sleep(1_000); + } + + stop.set(true); + + IgniteInternalFuture f; - lock.unlock(); + Exception err = null; + + while ((f = q.poll()) != null) { + try { + f.get(60_000); + } + catch (Exception e) { + error("Test operation failed: " + e, e); + + if (err == null) + err = e; + } + } + + if (err != null) + fail("Test operation failed, see log for details"); + } + finally { + stopAllGrids(); + } + } + + /** + * @throws Exception If failed. + */ + public void testLockNodeStop() throws Exception { + final int nodeCnt = 3; + int threadCnt = 2; + final int keys = 100; + + try { + final AtomicBoolean stop = new AtomicBoolean(false); + + Queue> q = new ArrayDeque<>(nodeCnt); + + for (int i = 0; i < nodeCnt; i++) { + final Ignite ignite = startGrid(i); + + IgniteInternalFuture f = GridTestUtils.runMultiThreadedAsync(new Runnable() { + @Override public void run() { + while (!Thread.currentThread().isInterrupted() && !stop.get()) { + try { + IgniteCache cache = ignite.cache(REPLICATED_TEST_CACHE); + + for (int i = 0; i < keys; i++) { + Lock lock = cache.lock(i); + lock.lock(); + + try { + cache.put(i, i); + } + finally { + lock.unlock(); + } + } + } + catch (Exception e) { + log.info("Ignore error: " + e); + + break; } } } @@ -176,12 +252,31 @@ public void testLockTopologyChange() throws Exception { U.sleep(1_000); } + U.sleep(ThreadLocalRandom.current().nextLong(500) + 500); + + // Stop all nodes, check that threads executing cache operations do not hang. + stopAllGrids(); + stop.set(true); IgniteInternalFuture f; - while ((f = q.poll()) != null) - f.get(2_000); + Exception err = null; + + while ((f = q.poll()) != null) { + try { + f.get(60_000); + } + catch (Exception e) { + error("Test operation failed: " + e, e); + + if (err == null) + err = e; + } + } + + if (err != null) + fail("Test operation failed, see log for details"); } finally { stopAllGrids(); From b07e055c4d53105a1a21446fefaad0e609ed196d Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 20 Dec 2018 17:44:55 +0300 Subject: [PATCH 263/403] IGNITE-10493 Refactor exchange timings measurement - Fixes #5688. Signed-off-by: Pavel Kovalenko (cherry picked from commit edcc1089aaa8efa7ede38af2a5dfdb0ef00b7bc5) Signed-off-by: Pavel Kovalenko --- .../cache/CacheAffinitySharedManager.java | 93 +++--- .../GridCachePartitionExchangeManager.java | 2 + .../GridDhtPartitionsExchangeFuture.java | 211 ++++++++---- .../GridDhtPartitionsSingleMessage.java | 55 ++- .../GridCacheDatabaseSharedManager.java | 10 +- .../ignite/internal/util/IgniteStopwatch.java | 230 +++++++++++++ .../ignite/internal/util/IgniteTicker.java | 52 +++ .../apache/ignite/internal/util/TimeBag.java | 312 ++++++++++++++++++ 8 files changed, 852 insertions(+), 113 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/IgniteStopwatch.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTicker.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index ce87fa94776e0..bf59b11a39251 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -798,15 +798,12 @@ public IgniteInternalFuture onCacheChangeRequest( ) throws IgniteCheckedException { assert exchActions != null && !exchActions.empty() : exchActions; - long time = System.currentTimeMillis(); - IgniteInternalFuture res = cachesRegistry.update(exchActions); // Affinity did not change for existing caches. onCustomMessageNoAffinityChange(fut, crd, exchActions); - if (log.isInfoEnabled()) - log.info("Updating caches registry performed in " + (System.currentTimeMillis() - time) + " ms."); + fut.timeBag().finishGlobalStage("Update caches registry"); processCacheStartRequests(fut, crd, exchActions); @@ -869,8 +866,6 @@ private void processCacheStartRequests( final ExchangeDiscoveryEvents evts = fut.context().events(); - long time = System.currentTimeMillis(); - Map startCacheInfos = new LinkedHashMap<>(); for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) { @@ -950,15 +945,11 @@ private void processCacheStartRequests( } } - if (log.isInfoEnabled()) - log.info("Caches starting performed in " + (System.currentTimeMillis() - time) + " ms."); - - time = System.currentTimeMillis(); + fut.timeBag().finishGlobalStage("Start caches"); initAffinityOnCacheGroupsStart(fut, exchActions, crd); - if (log.isInfoEnabled()) - log.info("Affinity initialization for started caches performed in " + (System.currentTimeMillis() - time) + " ms."); + fut.timeBag().finishGlobalStage("Affinity initialization on cache group start"); } /** @@ -994,6 +985,9 @@ private void initAffinityOnCacheGroupsStart( } } + fut.timeBag().finishLocalStage("Affinity initialization on cache group start " + + "[grp=" + grpDesc.cacheOrGroupName() + "]"); + return null; } ); @@ -1102,6 +1096,9 @@ public void onExchangeChangeAffinityMessage( newAssignment = idealAssignment; aff.initialize(topVer, cachedAssignment(aff, newAssignment, affCache)); + + exchFut.timeBag().finishLocalStage("Affinity recalculate by change affinity message " + + "[grp=" + aff.cacheOrGroupName() + "]"); } }); } @@ -1186,6 +1183,9 @@ public void onChangeAffinityMessage( aff.clientEventTopologyChange(exchFut.firstEvent(), topVer); cctx.exchange().exchangerUpdateHeartbeat(); + + exchFut.timeBag().finishLocalStage("Affinity change by custom message " + + "[grp=" + aff.cacheOrGroupName() + "]"); } }); } @@ -1365,6 +1365,9 @@ public IgniteInternalFuture initStartedCaches( calculateAndInit(fut.events(), cache.affinity(), fut.initialVersion()); cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Affinity initialization (crd, new cache) " + + "[grp=" + desc.cacheOrGroupName() + "]"); } } }); @@ -1376,6 +1379,9 @@ public IgniteInternalFuture initStartedCaches( initAffinity(cachesRegistry.group(aff.groupId()), aff, fut); cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Affinity initialization (new cache) " + + "[grp=" + aff.cacheOrGroupName() + "]"); } } }); @@ -1466,8 +1472,6 @@ public void applyAffinityFromFullMessage( final Map>> affCache = new ConcurrentHashMap<>(); - long time = System.currentTimeMillis(); - forAllCacheGroups(false, new IgniteInClosureX() { @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException { ExchangeDiscoveryEvents evts = fut.context().events(); @@ -1499,11 +1503,11 @@ public void applyAffinityFromFullMessage( newAssignment = idealAssignment; aff.initialize(evts.topologyVersion(), cachedAssignment(aff, newAssignment, affCache)); + + fut.timeBag().finishLocalStage("Affinity applying from full message " + + "[grp=" + aff.cacheOrGroupName() + "]"); } }); - - if (log.isInfoEnabled()) - log.info("Affinity applying from full message performed in " + (System.currentTimeMillis() - time) + " ms."); } /** @@ -1528,8 +1532,6 @@ public void onLocalJoin( final Map nodesByOrder = new ConcurrentHashMap<>(); - long time = System.currentTimeMillis(); - forAllCacheGroups(false, new IgniteInClosureX() { @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException { ExchangeDiscoveryEvents evts = fut.context().events(); @@ -1567,11 +1569,11 @@ else if (fut.cacheGroupAddedOnExchange(aff.groupId(), grp.receivedFrom())) calculateAndInit(evts, aff, evts.topologyVersion()); grp.topology().initPartitionsWhenAffinityReady(resTopVer, fut); + + fut.timeBag().finishLocalStage("Affinity initialization (local join) " + + "[grp=" + grp.cacheOrGroupName() + "]"); } }); - - if (log.isInfoEnabled()) - log.info("Affinity initialization on local join performed in " + (System.currentTimeMillis() - time) + " ms."); } /** @@ -1586,8 +1588,6 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur assert fut.context().mergeExchanges(); assert evts.hasServerJoin() && !evts.hasServerLeft(); - long time = System.currentTimeMillis(); - WaitRebalanceInfo waitRebalanceInfo = initAffinityOnNodeJoin(fut, crd); this.waitInfo = waitRebalanceInfo != null && !waitRebalanceInfo.empty() ? waitRebalanceInfo : null; @@ -1600,10 +1600,6 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur ", waitGrps=" + (info != null ? groupNames(info.waitGrps.keySet()) : null) + ']'); } } - - if (log.isInfoEnabled()) - log.info("Affinity recalculation (on server join) performed in " - + (System.currentTimeMillis() - time) + " ms."); } /** @@ -1614,8 +1610,6 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur public Map onServerLeftWithExchangeMergeProtocol( final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException { - long time = System.currentTimeMillis(); - final ExchangeDiscoveryEvents evts = fut.context().events(); assert fut.context().mergeExchanges(); @@ -1623,10 +1617,6 @@ public Map onServerLeftWithExchangeMergeProt Map result = onReassignmentEnforced(fut); - if (log.isInfoEnabled()) - log.info("Affinity recalculation (on server left) performed in " - + (System.currentTimeMillis() - time) + " ms."); - return result; } @@ -1642,14 +1632,8 @@ public Map onCustomEventWithEnforcedAffinity { assert DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent()); - long time = System.currentTimeMillis(); - Map result = onReassignmentEnforced(fut); - if (log.isInfoEnabled()) - log.info("Affinity recalculation (custom message) performed in " - + (System.currentTimeMillis() - time) + " ms."); - return result; } @@ -1675,6 +1659,9 @@ private Map onReassignmentEnforced( if (!cache.rebalanceEnabled || fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom())) cache.affinity().initialize(topVer, assign); + + fut.timeBag().finishLocalStage("Affinity initialization (enforced) " + + "[grp=" + desc.cacheOrGroupName() + "]"); } }); @@ -1712,11 +1699,17 @@ public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd) calculateAndInit(fut.events(), grpHolder.affinity(), topVer); cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("First node affinity initialization (node join) " + + "[grp=" + desc.cacheOrGroupName() + "]"); } }); } - else + else { fetchAffinityOnJoin(fut); + + fut.timeBag().finishLocalStage("Affinity fetch"); + } } else waitRebalanceInfo = initAffinityOnNodeJoin(fut, crd); @@ -1927,6 +1920,9 @@ public boolean onCentralizedAffinityChange(final GridDhtPartitionsExchangeFuture cache.aff.calculate(fut.initialVersion(), fut.events(), fut.events().discoveryCache()); cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Affinity centralized initialization (crd) " + + "[grp=" + desc.cacheOrGroupName() + "]"); } }); } @@ -1936,6 +1932,9 @@ public boolean onCentralizedAffinityChange(final GridDhtPartitionsExchangeFuture aff.calculate(fut.initialVersion(), fut.events(), fut.events().discoveryCache()); cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Affinity centralized initialization " + + "[grp=" + aff.cacheOrGroupName() + "]"); } }); } @@ -2057,6 +2056,9 @@ public IgniteInternalFuture initCoordinatorCaches( assert old == null : old; cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Coordinator affinity cache init " + + "[grp=" + desc.cacheOrGroupName() + "]"); } }); @@ -2135,6 +2137,9 @@ private CacheGroupHolder groupHolder(AffinityTopologyVersion topVer, final Cache affCache); cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Affinity initialization (node join) " + + "[grp=" + grp.cacheOrGroupName() + "]"); } }); @@ -2173,6 +2178,9 @@ private CacheGroupHolder groupHolder(AffinityTopologyVersion topVer, final Cache } cctx.exchange().exchangerUpdateHeartbeat(); + + fut.timeBag().finishLocalStage("Affinity initialization (crd, node join) " + + "[grp=" + desc.cacheOrGroupName() + "]"); } }); @@ -2518,6 +2526,9 @@ else if (curPrimary != null && !curPrimary.equals(newPrimary)) { if (initAff) grpHolder.affinity().initialize(topVer, newAssignment0); + + fut.timeBag().finishLocalStage("Affinity recalculation (partitions availability) " + + "[grp=" + desc.cacheOrGroupName() + "]"); } }); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 453b228b02379..28f96d9ecfcf4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -2864,6 +2864,8 @@ else if (task instanceof ForceRebalanceExchangeTask) { } } + exchFut.timeBag().finishGlobalStage("Waiting in exchange queue"); + exchFut.init(newCrd); int dumpCnt = 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 077625678e8c2..f86b11c10a47c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -100,6 +100,7 @@ import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.TimeBag; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -337,6 +338,15 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte /** Future for wait all exchange listeners comepleted. */ private final GridFutureAdapter afterLsnrCompleteFut = new GridFutureAdapter<>(); + /** Time bag to measure and store exchange stages times. */ + private final TimeBag timeBag; + + /** Start time of exchange. */ + private long startTime = System.nanoTime(); + + /** Discovery lag / Clocks discrepancy, calculated on coordinator when all single messages are received. */ + private T2 discoveryLag; + /** * @param cctx Cache context. * @param busyLock Busy lock. @@ -368,6 +378,8 @@ public GridDhtPartitionsExchangeFuture( log = cctx.logger(getClass()); exchLog = cctx.logger(EXCHANGE_LOG); + timeBag = new TimeBag(); + initFut = new GridFutureAdapter() { @Override public IgniteLogger logger() { return log; @@ -661,6 +673,13 @@ private void initCoordinatorCaches(boolean newCrd) throws IgniteCheckedException } } + /** + * @return Object to collect exchange timings. + */ + public TimeBag timeBag() { + return timeBag; + } + /** * Starts activity. * @@ -718,6 +737,8 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException { ", allowMerge=" + exchCtx.mergeExchanges() + ']'); } + timeBag.finishGlobalStage("Exchange parameters initialization"); + ExchangeType exchange; if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) { @@ -802,6 +823,8 @@ else if (msg instanceof WalStateAbstractMessage) updateTopologies(crdNode); + timeBag.finishGlobalStage("Determine exchange type"); + switch (exchange) { case ALL: { distributedExchange(); @@ -885,6 +908,8 @@ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedExce finally { cctx.exchange().exchangerBlockingSectionEnd(); } + + timeBag.finishGlobalStage("Baseline change callback"); } cctx.exchange().exchangerBlockingSectionBegin(); @@ -896,6 +921,8 @@ private IgniteInternalFuture initCachesOnLocalJoin() throws IgniteCheckedExce cctx.exchange().exchangerBlockingSectionEnd(); } + timeBag.finishGlobalStage("Components activation"); + IgniteInternalFuture cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), exchActions == null ? null : exchActions.localJoinContext()); @@ -1344,6 +1371,8 @@ private void distributedExchange() throws IgniteCheckedException { } } + timeBag.finishGlobalStage("Preloading notification"); + cctx.exchange().exchangerBlockingSectionBegin(); try { @@ -1356,6 +1385,8 @@ private void distributedExchange() throws IgniteCheckedException { cctx.exchange().exchangerBlockingSectionEnd(); } + timeBag.finishGlobalStage("WAL history reservation"); + // Skipping wait on local join is available when all cluster nodes have the same protocol. boolean skipWaitOnLocalJoin = cctx.exchange().latch().canSkipJoiningNodes(initialVersion()) && localJoinExchange(); @@ -1446,6 +1477,8 @@ private void distributedExchange() throws IgniteCheckedException { } } + timeBag.finishGlobalStage("After states restored callback"); + changeWalModeIfNeeded(); if (events().hasServerLeft()) @@ -1455,8 +1488,11 @@ private void distributedExchange() throws IgniteCheckedException { try { if (crd.isLocal()) { - if (remaining.isEmpty()) + if (remaining.isEmpty()) { + initFut.onDone(true); + onAllReceived(null); + } } else sendPartitions(crd); @@ -1678,6 +1714,8 @@ private void waitPartitionRelease(boolean distributed, boolean doRollback) throw } } + timeBag.finishGlobalStage("Wait partitions release"); + if (releaseLatch == null) { assert !distributed : "Partitions release latch must be initialized in distributed mode."; @@ -1718,6 +1756,8 @@ private void waitPartitionRelease(boolean distributed, boolean doRollback) throw catch (IgniteCheckedException e) { U.warn(log, "Stop waiting for partitions release latch: " + e.getMessage()); } + + timeBag.finishGlobalStage("Wait partitions release latch"); } /** @@ -1792,8 +1832,6 @@ public boolean localJoinExchange() { private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException { assert node != null; - long time = System.currentTimeMillis(); - GridDhtPartitionsSingleMessage msg; // Reset lost partitions before sending local partitions to coordinator. @@ -1828,6 +1866,8 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException else if (localJoinExchange()) msg.cacheGroupsAffinityRequest(exchCtx.groupsAffinityRequestOnJoin()); + msg.exchangeStartTime(startTime); + if (log.isTraceEnabled()) log.trace("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + msg + ']'); @@ -1838,9 +1878,6 @@ else if (localJoinExchange()) if (log.isDebugEnabled()) log.debug("Node left during partition exchange [nodeId=" + node.id() + ", exchId=" + exchId + ']'); } - - if (log.isInfoEnabled()) - log.info("Sending Single Message performed in " + (System.currentTimeMillis() - time) + " ms."); } /** @@ -1996,6 +2033,28 @@ public void finishMerged() { super.onDone(null, null); } + /** + * Make a log message that contains given exchange timings. + * + * @param header Header of log message. + * @param timings Exchange stages timings. + * @return Log message with exchange timings and exchange version. + */ + private String exchangeTimingsLogMessage(String header, List timings) { + StringBuilder timingsToLog = new StringBuilder(); + + timingsToLog.append(header).append(" ["); + timingsToLog.append("startVer=").append(initialVersion()); + timingsToLog.append(", resVer=").append(topologyVersion()); + + for (String stageTiming : timings) + timingsToLog.append(", ").append(stageTiming); + + timingsToLog.append(']'); + + return timingsToLog.toString(); + } + /** {@inheritDoc} */ @Override public boolean onDone(@Nullable AffinityTopologyVersion res, @Nullable Throwable err) { assert res != null || err != null : "TopVer=" + res + ", err=" + err; @@ -2134,14 +2193,26 @@ public void finishMerged() { if (super.onDone(res, err)) { afterLsnrCompleteFut.onDone(); - if (log.isDebugEnabled()) { - log.debug("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange= " + this + - ", durationFromInit=" + (U.currentTimeMillis() - initTs) + ']'); - } - else if (log.isInfoEnabled()) { + if (log.isInfoEnabled()) { log.info("Completed partition exchange [localNode=" + cctx.localNodeId() + - ", exchange=" + shortInfo() + ", topVer=" + topologyVersion() + - ", durationFromInit=" + (U.currentTimeMillis() - initTs) + ']'); + ", exchange=" + (log.isDebugEnabled() ? this : shortInfo()) + ", topVer=" + topologyVersion() + "]"); + + if (err == null) { + timeBag.finishGlobalStage("Exchange done"); + + // Collect all stages timings. + List timings = timeBag.stagesTimings(); + + if (discoveryLag != null && discoveryLag.get1() != 0) + timings.add("Discovery lag=" + discoveryLag.get1() + + " ms, Latest started node id=" + discoveryLag.get2()); + + log.info(exchangeTimingsLogMessage("Exchange timings", timings)); + + List localTimings = timeBag.longestLocalStagesTimings(3); + + log.info(exchangeTimingsLogMessage("Exchange longest local stages", localTimings)); + } } initFut.onDone(err == null); @@ -2178,6 +2249,42 @@ else if (log.isInfoEnabled()) { return false; } + /** + * Calculates discovery lag (Maximal difference between exchange start times across all nodes). + * + * @param declared Single messages that were expected to be received during exchange. + * @param merged Single messages from nodes that were merged during exchange. + * + * @return Pair with discovery lag and node id which started exchange later than others. + */ + private T2 calculateDiscoveryLag( + Map declared, + Map merged + ) { + Map msgs = new HashMap<>(declared); + + msgs.putAll(merged); + + long minStartTime = startTime; + long maxStartTime = startTime; + UUID latestStartedNode = cctx.localNodeId(); + + for (Map.Entry msg : msgs.entrySet()) { + UUID nodeId = msg.getKey(); + long exchangeTime = msg.getValue().exchangeStartTime(); + + if (exchangeTime != 0) { + minStartTime = Math.min(minStartTime, exchangeTime); + maxStartTime = Math.max(maxStartTime, exchangeTime); + } + + if (maxStartTime == exchangeTime) + latestStartedNode = nodeId; + } + + return new T2<>(TimeUnit.NANOSECONDS.toMillis(maxStartTime - minStartTime), latestStartedNode); + } + /** * @param exchangeActions Exchange actions. * @return Map of cache names and start descriptors. @@ -2747,6 +2854,7 @@ else if (log.isDebugEnabled()) } } } + if (allReceived) { if (!awaitSingleMapUpdates()) return; @@ -2981,8 +3089,6 @@ else if (cntr == maxCntr.cnt) private void detectLostPartitions(AffinityTopologyVersion resTopVer) { boolean detected = false; - long time = System.currentTimeMillis(); - synchronized (cctx.exchange().interruptLock()) { if (Thread.currentThread().isInterrupted()) return; @@ -3007,8 +3113,7 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { cctx.exchange().scheduleResendPartitions(); } - if (log.isInfoEnabled()) - log.info("Detecting lost partitions performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Detect lost partitions"); } /** @@ -3104,6 +3209,10 @@ private void sendExchangeFailureMessage() { */ private void onAllReceived(@Nullable Collection sndResNodes) { try { + initFut.get(); + + timeBag.finishGlobalStage("Waiting for all single messages"); + assert crd.isLocal(); assert partHistSuppliers.isEmpty() : partHistSuppliers; @@ -3127,12 +3236,9 @@ private void onAllReceived(@Nullable Collection sndResNodes) { if (log.isInfoEnabled()) log.info("Coordinator received all messages, try merge [ver=" + initialVersion() + ']'); - long time = System.currentTimeMillis(); - boolean finish = cctx.exchange().mergeExchangesOnCoordinator(this); - if (log.isInfoEnabled()) - log.info("Exchanges merging performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Exchanges merge"); if (!finish) return; @@ -3168,8 +3274,6 @@ private void finishExchangeOnCoordinator(@Nullable Collection sndRe Map idealAffDiff = null; - long time = System.currentTimeMillis(); - if (exchCtx.mergeExchanges()) { synchronized (mux) { if (mergedJoinExchMsgs != null) { @@ -3203,8 +3307,7 @@ private void finishExchangeOnCoordinator(@Nullable Collection sndRe } } - if (log.isInfoEnabled()) - log.info("Affinity changes (coordinator) applied in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Affinity recalculation (crd)"); Map joinedNodeAff = null; @@ -3237,6 +3340,8 @@ private void finishExchangeOnCoordinator(@Nullable Collection sndRe } } + timeBag.finishGlobalStage("Collect update counters and create affinity messages"); + validatePartitionsState(); if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) { @@ -3270,22 +3375,25 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage } // Recalculate new affinity based on partitions availability. - if (!exchCtx.mergeExchanges() && forceAffReassignment) + if (!exchCtx.mergeExchanges() && forceAffReassignment) { idealAffDiff = cctx.affinity().onCustomEventWithEnforcedAffinityReassignment(this); + timeBag.finishGlobalStage("Ideal affinity diff calculation (enforced)"); + } + for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) { if (!grpCtx.isLocal()) grpCtx.topology().applyUpdateCounters(); } + timeBag.finishGlobalStage("Apply update counters"); + updateLastVersion(cctx.versions().last()); cctx.versions().onExchange(lastVer.get().order()); IgniteProductVersion minVer = exchCtx.events().discoveryCache().minimumNodeVersion(); - time = System.currentTimeMillis(); - GridDhtPartitionsFullMessage msg = createPartitionsMessage(true, minVer.compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0); @@ -3302,8 +3410,7 @@ else if (forceAffReassignment) msg.prepareMarshal(cctx); - if (log.isInfoEnabled()) - log.info("Preparing Full Message performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Full message preparing"); synchronized (mux) { finishState = new FinishState(crd.id(), resTopVer, msg); @@ -3314,8 +3421,6 @@ else if (forceAffReassignment) if (centralizedAff) { assert !exchCtx.mergeExchanges(); - time = System.currentTimeMillis(); - IgniteInternalFuture>>> fut = cctx.affinity().initAffinityOnNodeLeft(this); if (!fut.isDone()) { @@ -3327,9 +3432,6 @@ else if (forceAffReassignment) } else onAffinityInitialized(fut); - - if (log.isInfoEnabled()) - log.info("Centralized affinity changes are performed in " + (System.currentTimeMillis() - time) + " ms."); } else { Set nodes; @@ -3353,6 +3455,8 @@ else if (forceAffReassignment) } } } + else + mergedJoinExchMsgs0 = Collections.emptyMap(); if (!F.isEmpty(sndResNodes)) nodes.addAll(sndResNodes); @@ -3361,6 +3465,13 @@ else if (forceAffReassignment) if (!nodes.isEmpty()) sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff); + timeBag.finishGlobalStage("Full message sending"); + + discoveryLag = calculateDiscoveryLag( + msgs, + mergedJoinExchMsgs0 + ); + partitionsSent = true; if (!stateChangeExchange()) @@ -3417,6 +3528,8 @@ else if (forceAffReassignment) cctx.discovery().sendCustomEvent(stateFinishMsg); + timeBag.finishGlobalStage("State finish message sending"); + if (!centralizedAff) onDone(exchCtx.events().topologyVersion(), null); } @@ -3455,8 +3568,6 @@ private List nonLocalCacheGroups() { * Validates that partition update counters and cache sizes for all caches are consistent. */ private void validatePartitionsState() { - long time = System.currentTimeMillis(); - try { U.doInParallel( cctx.kernalContext().getSystemExecutorService(), @@ -3496,16 +3607,13 @@ private void validatePartitionsState() { throw new IgniteException("Failed to validate partitions state", e); } - if (log.isInfoEnabled()) - log.info("Partitions validation performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Validate partitions states"); } /** * */ private void assignPartitionsStates() { - long time = System.currentTimeMillis(); - try { U.doInParallel( cctx.kernalContext().getSystemExecutorService(), @@ -3530,8 +3638,7 @@ private void assignPartitionsStates() { throw new IgniteException("Failed to assign partition states", e); } - if (log.isInfoEnabled()) - log.info("Partitions assignment performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Assign partitions states"); } /** @@ -3542,8 +3649,6 @@ private void finalizePartitionCounters() { // Reserve at least 2 threads for system operations. int parallelismLvl = U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2); - long time = System.currentTimeMillis(); - try { U.doInParallel( parallelismLvl, @@ -3560,8 +3665,7 @@ private void finalizePartitionCounters() { throw new IgniteException("Failed to finalize partition counters", e); } - if (log.isInfoEnabled()) - log.info("Partition counters finalization performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Finalize update counters"); } /** @@ -3804,6 +3908,8 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti assert exchId.equals(msg.exchangeId()) : msg; assert msg.lastVersion() != null : msg; + timeBag.finishGlobalStage("Waiting for Full message"); + if (checkCrd) { assert node != null; @@ -3879,8 +3985,6 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti AffinityTopologyVersion resTopVer = initialVersion(); - long time = System.currentTimeMillis(); - if (exchCtx.mergeExchanges()) { if (msg.resultTopologyVersion() != null && !initialVersion().equals(msg.resultTopologyVersion())) { if (log.isInfoEnabled()) { @@ -3924,8 +4028,7 @@ else if (localJoinExchange() && !exchCtx.fetchAffinityOnJoin()) else if (forceAffReassignment) cctx.affinity().applyAffinityFromFullMessage(this, msg); - if (log.isInfoEnabled()) - log.info("Affinity changes applied in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Affinity recalculation"); if (dynamicCacheStartExchange() && !F.isEmpty(exchangeGlobalExceptions)) { assert cctx.localNode().isClient(); @@ -3967,8 +4070,6 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa partHistSuppliers.putAll(msg.partitionHistorySuppliers()); - long time = System.currentTimeMillis(); - // Reserve at least 2 threads for system operations. int parallelismLvl = U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2); @@ -4017,9 +4118,7 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa partitionsReceived = true; - if (log.isInfoEnabled()) - log.info("Full map updating for " + msg.partitions().size() - + " groups performed in " + (System.currentTimeMillis() - time) + " ms."); + timeBag.finishGlobalStage("Full map updating"); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index 8db717651b298..3316e3ec432ee 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -102,6 +102,9 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes @GridDirectCollection(Integer.class) private Collection grpsAffRequest; + /** Start time of exchange on node which sent this message in nanoseconds. */ + private long exchangeStartTime; + /** * Exchange finish message, sent to new coordinator when it tries to * restore state after previous coordinator failed during exchange. @@ -316,6 +319,20 @@ public void setError(Exception ex) { return err; } + /** + * Start time of exchange on node which sent this message. + */ + public long exchangeStartTime() { + return exchangeStartTime; + } + + /** + * @param exchangeStartTime Start time of exchange. + */ + public void exchangeStartTime(long exchangeStartTime) { + this.exchangeStartTime = exchangeStartTime; + } + /** {@inheritDoc} * @param ctx*/ @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException { @@ -474,40 +491,47 @@ public void setError(Exception ex) { writer.incrementState(); case 9: - if (!writer.writeMessage("finishMsg", finishMsg)) + if (!writer.writeLong("exchangeStartTime", exchangeStartTime)) return false; writer.incrementState(); case 10: - if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT)) + if (!writer.writeMessage("finishMsg", finishMsg)) return false; writer.incrementState(); case 11: - if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes)) + if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT)) return false; writer.incrementState(); case 12: - if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes)) + if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes)) return false; writer.incrementState(); case 13: - if (!writer.writeByteArray("partsBytes", partsBytes)) + if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes)) return false; writer.incrementState(); case 14: + if (!writer.writeByteArray("partsBytes", partsBytes)) + return false; + + writer.incrementState(); + + case 15: if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes)) return false; writer.incrementState(); + } return true; @@ -549,7 +573,7 @@ public void setError(Exception ex) { reader.incrementState(); case 9: - finishMsg = reader.readMessage("finishMsg"); + exchangeStartTime = reader.readLong("exchangeStartTime"); if (!reader.isLastRead()) return false; @@ -557,7 +581,7 @@ public void setError(Exception ex) { reader.incrementState(); case 10: - grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT); + finishMsg = reader.readMessage("finishMsg"); if (!reader.isLastRead()) return false; @@ -565,7 +589,7 @@ public void setError(Exception ex) { reader.incrementState(); case 11: - partCntrsBytes = reader.readByteArray("partCntrsBytes"); + grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT); if (!reader.isLastRead()) return false; @@ -573,7 +597,7 @@ public void setError(Exception ex) { reader.incrementState(); case 12: - partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes"); + partCntrsBytes = reader.readByteArray("partCntrsBytes"); if (!reader.isLastRead()) return false; @@ -581,7 +605,7 @@ public void setError(Exception ex) { reader.incrementState(); case 13: - partsBytes = reader.readByteArray("partsBytes"); + partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes"); if (!reader.isLastRead()) return false; @@ -589,12 +613,21 @@ public void setError(Exception ex) { reader.incrementState(); case 14: + partsBytes = reader.readByteArray("partsBytes"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 15: partsSizesBytes = reader.readByteArray("partsSizesBytes"); if (!reader.isLastRead()) return false; reader.incrementState(); + } return reader.afterMessageRead(GridDhtPartitionsSingleMessage.class); @@ -607,7 +640,7 @@ public void setError(Exception ex) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 15; + return 16; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 1d89b83ccf4a8..336aa427e4320 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -1303,6 +1303,9 @@ private void shutdownCheckpointer(boolean cancel) { if (cacheGroup.localStartVersion().equals(fut.initialVersion())) cacheGroup.topology().afterStateRestored(fut.initialVersion()); + + fut.timeBag().finishLocalStage("Restore partition states " + + "[grp=" + cacheGroup.cacheOrGroupName() + "]"); } finally { cctx.database().checkpointReadUnlock(); @@ -1311,6 +1314,8 @@ private void shutdownCheckpointer(boolean cancel) { return null; } ); + + fut.timeBag().finishGlobalStage("Restore partition states"); } if (cctx.kernalContext().query().moduleEnabled()) { @@ -1988,8 +1993,6 @@ private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { * @throws IgniteCheckedException If first checkpoint has failed. */ @Override public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException { - long time = System.currentTimeMillis(); - IgniteThread cpThread = new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer); cpThread.start(); @@ -2000,9 +2003,6 @@ private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException { if (chp != null) chp.cpBeginFut.get(); - - if (log.isInfoEnabled()) - log.info("Checkpointer initilialzation performed in " + (System.currentTimeMillis() - time) + " ms."); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteStopwatch.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteStopwatch.java new file mode 100644 index 0000000000000..83cd7bc8d3ba6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteStopwatch.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2008 The Guava Authors + */ + +package org.apache.ignite.internal.util; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import org.jetbrains.annotations.NotNull; + +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +/** + * An object that measures elapsed time in nanoseconds. It is useful to measure elapsed time using + * this class instead of direct calls to {@link System#nanoTime} for a few reasons: + * + *
    + *
  • An alternate time source can be substituted, for testing or performance reasons. + *
  • As documented by {@code nanoTime}, the value returned has no absolute meaning, and can only + * be interpreted as relative to another timestamp returned by {@code nanoTime} at a different + * time. {@code Stopwatch} is a more effective abstraction because it exposes only these + * relative values, not the absolute ones. + *
+ * + *

Basic usage: + * + *

{@code
+ * Stopwatch stopwatch = Stopwatch.createStarted();
+ * doSomething();
+ * stopwatch.stop(); // optional
+ *
+ * Duration duration = stopwatch.elapsed();
+ *
+ * log.info("time: " + stopwatch); // formatted string like "12.3 ms"
+ * }
+ * + *

Stopwatch methods are not idempotent; it is an error to start or stop a stopwatch that is + * already in the desired state. + * + *

When testing code that uses this class, use {@link #createUnstarted(IgniteTicker)} or {@link + * #createStarted(IgniteTicker)} to supply a fake or mock ticker. This allows you to simulate any valid + * behavior of the stopwatch. + * + *

Note: This class is not thread-safe. + * + *

Warning for Android users: a stopwatch with default behavior may not continue to keep + * time while the device is asleep. Instead, create one like this: + * + *

{@code
+ * Stopwatch.createStarted(
+ *      new Ticker() {
+ *        public long read() {
+ *          return android.os.SystemClock.elapsedRealtimeNanos();
+ *        }
+ *      });
+ * }
+ */ +@SuppressWarnings("GoodTime") // lots of violations +public final class IgniteStopwatch { + /** Ticker. */ + private final IgniteTicker ticker; + /** Is running. */ + private boolean isRunning; + /** Elapsed nanos. */ + private long elapsedNanos; + /** Start tick. */ + private long startTick; + + /** + * Creates (but does not start) a new stopwatch using {@link System#nanoTime} as its time source. + */ + public static IgniteStopwatch createUnstarted() { + return new IgniteStopwatch(); + } + + /** + * Creates (but does not start) a new stopwatch, using the specified time source. + */ + public static IgniteStopwatch createUnstarted(IgniteTicker ticker) { + return new IgniteStopwatch(ticker); + } + + /** + * Creates (and starts) a new stopwatch using {@link System#nanoTime} as its time source. + */ + public static IgniteStopwatch createStarted() { + return new IgniteStopwatch().start(); + } + + /** + * Creates (and starts) a new stopwatch, using the specified time source. + */ + public static IgniteStopwatch createStarted(IgniteTicker ticker) { + return new IgniteStopwatch(ticker).start(); + } + + /** + * Default constructor. + */ + IgniteStopwatch() { + this.ticker = IgniteTicker.systemTicker(); + } + + /** + * @param ticker Ticker. + */ + IgniteStopwatch(@NotNull IgniteTicker ticker) { + this.ticker = ticker; + } + + /** + * Returns {@code true} if {@link #start()} has been called on this stopwatch, and {@link #stop()} + * has not been called since the last call to {@code start()}. + */ + public boolean isRunning() { + return isRunning; + } + + /** + * Starts the stopwatch. + * + * @return this {@code Stopwatch} instance + * @throws IllegalStateException if the stopwatch is already running. + */ + public IgniteStopwatch start() { + assert !isRunning : "This stopwatch is already running."; + + isRunning = true; + + startTick = ticker.read(); + + return this; + } + + /** + * Stops the stopwatch. Future reads will return the fixed duration that had elapsed up to this + * point. + * + * @return this {@code Stopwatch} instance + * @throws IllegalStateException if the stopwatch is already stopped. + */ + public IgniteStopwatch stop() { + long tick = ticker.read(); + + assert !isRunning : "This stopwatch is already running."; + + isRunning = false; + elapsedNanos += tick - startTick; + return this; + } + + /** + * Sets the elapsed time for this stopwatch to zero, and places it in a stopped state. + * + * @return this {@code Stopwatch} instance + */ + public IgniteStopwatch reset() { + elapsedNanos = 0; + + isRunning = false; + + return this; + } + + /** + * + */ + private long elapsedNanos() { + return isRunning ? ticker.read() - startTick + elapsedNanos : elapsedNanos; + } + + /** + * Returns the current elapsed time shown on this stopwatch, expressed in the desired time unit, + * with any fraction rounded down. + * + *

Note: the overhead of measurement can be more than a microsecond, so it is generally + * not useful to specify {@link TimeUnit#NANOSECONDS} precision here. + * + *

It is generally not a good idea to use an ambiguous, unitless {@code long} to represent + * elapsed time. Therefore, we recommend using {@link #elapsed()} instead, which returns a + * strongly-typed {@link Duration} instance. + */ + public long elapsed(TimeUnit desiredUnit) { + return desiredUnit.convert(elapsedNanos(), NANOSECONDS); + } + + /** + * Returns the current elapsed time shown on this stopwatch as a {@link Duration}. Unlike {@link + * #elapsed(TimeUnit)}, this method does not lose any precision due to rounding. + */ + public Duration elapsed() { + return Duration.ofNanos(elapsedNanos()); + } + + /** + * @param nanos Nanos. + */ + private static TimeUnit chooseUnit(long nanos) { + if (DAYS.convert(nanos, NANOSECONDS) > 0) return DAYS; + if (HOURS.convert(nanos, NANOSECONDS) > 0) return HOURS; + if (MINUTES.convert(nanos, NANOSECONDS) > 0) return MINUTES; + if (SECONDS.convert(nanos, NANOSECONDS) > 0) return SECONDS; + if (MILLISECONDS.convert(nanos, NANOSECONDS) > 0) return MILLISECONDS; + if (MICROSECONDS.convert(nanos, NANOSECONDS) > 0) return MICROSECONDS; + return NANOSECONDS; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTicker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTicker.java new file mode 100644 index 0000000000000..1f7f41d9a8d76 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTicker.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2011 The Guava Authors + */ + +package org.apache.ignite.internal.util; + +/** + * A time source; returns a time value representing the number of nanoseconds elapsed since some + * fixed but arbitrary point in time. Note that most users should use {@link IgniteStopwatch} instead of + * interacting with this class directly. + * + *

Warning: this interface can only be used to measure elapsed time, not wall time. + */ +public abstract class IgniteTicker { + /** Constructor for use by subclasses. */ + protected IgniteTicker() {} + + /** Returns the number of nanoseconds elapsed since this ticker's fixed point of reference. */ + public abstract long read(); + + /** + * A ticker that reads the current time using {@link System#nanoTime}. + */ + public static IgniteTicker systemTicker() { + return SYSTEM_TICKER; + } + + /** System ticker. */ + private static final IgniteTicker SYSTEM_TICKER = + new IgniteTicker() { + @Override public long read() { + return System.nanoTime(); + } + }; +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java b/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java new file mode 100644 index 0000000000000..0da1ee7ad4611 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.jetbrains.annotations.NotNull; + +/** + * Utility class to measure and collect timings of some execution workflow. + */ +public class TimeBag { + /** Initial global stage. */ + private final CompositeStage INITIAL_STAGE = new CompositeStage("", 0, new HashMap<>()); + + /** Lock. */ + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + /** Global stopwatch. */ + private final IgniteStopwatch globalStopwatch = IgniteStopwatch.createStarted(); + + /** Measurement unit. */ + private final TimeUnit measurementUnit; + + /** List of global stages (guarded by {@code lock}). */ + private final List stages; + + /** List of current local stages separated by threads (guarded by {@code lock}). */ + private Map> localStages; + + /** Last seen global stage by thread. */ + private final ThreadLocal tlLastSeenStage = ThreadLocal.withInitial(() -> INITIAL_STAGE); + + /** Thread-local stopwatch. */ + private final ThreadLocal tlStopwatch = ThreadLocal.withInitial(IgniteStopwatch::createUnstarted); + + + /** + * Default constructor. + */ + public TimeBag() { + this(TimeUnit.MILLISECONDS); + } + + /** + * @param measurementUnit Measurement unit. + */ + public TimeBag(TimeUnit measurementUnit) { + this.stages = new ArrayList<>(); + this.localStages = new ConcurrentHashMap<>(); + this.measurementUnit = measurementUnit; + + this.stages.add(INITIAL_STAGE); + } + + /** + * + */ + private CompositeStage lastCompletedGlobalStage() { + assert !stages.isEmpty() : "No stages :("; + + return stages.get(stages.size() - 1); + } + + /** + * @param description Description. + */ + public void finishGlobalStage(String description) { + lock.writeLock().lock(); + + try { + stages.add( + new CompositeStage(description, globalStopwatch.elapsed(measurementUnit), Collections.unmodifiableMap(localStages)) + ); + + localStages = new ConcurrentHashMap<>(); + + globalStopwatch.reset().start(); + } + finally { + lock.writeLock().unlock(); + } + } + + /** + * @param description Description. + */ + public void finishLocalStage(String description) { + lock.readLock().lock(); + + try { + CompositeStage lastSeen = tlLastSeenStage.get(); + CompositeStage lastCompleted = lastCompletedGlobalStage(); + IgniteStopwatch localStopWatch = tlStopwatch.get(); + + Stage stage; + + // We see this stage first time, get elapsed time from last completed global stage and start tracking local. + if (lastSeen != lastCompleted) { + stage = new Stage(description, globalStopwatch.elapsed(measurementUnit)); + + tlLastSeenStage.set(lastCompleted); + } + else + stage = new Stage(description, localStopWatch.elapsed(measurementUnit)); + + localStopWatch.reset().start(); + + // Associate local stage with current thread name. + String threadName = Thread.currentThread().getName(); + + localStages.computeIfAbsent(threadName, t -> new ArrayList<>()).add(stage); + } + finally { + lock.readLock().unlock(); + } + } + + /** + * @return Short name of desired measurement unit. + */ + private String measurementUnitShort() { + switch (measurementUnit) { + case MILLISECONDS: + return "ms"; + case SECONDS: + return "s"; + case NANOSECONDS: + return "ns"; + case MICROSECONDS: + return "mcs"; + case HOURS: + return "h"; + case MINUTES: + return "min"; + case DAYS: + return "days"; + default: + return ""; + } + } + + /** + * @return List of string representation of all stage timings. + */ + public List stagesTimings() { + lock.readLock().lock(); + + try { + List timings = new ArrayList<>(); + + long totalTime = 0; + + // Skip initial stage. + for (int i = 1; i < stages.size(); i++) { + CompositeStage stage = stages.get(i); + + totalTime += stage.time(); + + timings.add(stage.toString()); + } + + // Add last stage with summary time of all global stages. + timings.add(new Stage("Total time", totalTime).toString()); + + return timings; + } + finally { + lock.readLock().unlock(); + } + } + + /** + * @param maxPerCompositeStage Max count of local stages to collect per composite stage. + * @return List of string represenation of longest local stages per each composite stage. + */ + public List longestLocalStagesTimings(int maxPerCompositeStage) { + lock.readLock().lock(); + + try { + List timings = new ArrayList<>(); + + for (int i = 1; i < stages.size(); i++) { + CompositeStage stage = stages.get(i); + + if (!stage.localStages.isEmpty()) { + PriorityQueue stagesByTime = new PriorityQueue<>(); + + for (Map.Entry> threadAndStages : stage.localStages.entrySet()) { + for (Stage locStage : threadAndStages.getValue()) + stagesByTime.add(locStage); + } + + int stageCount = 0; + while (!stagesByTime.isEmpty() && stageCount < maxPerCompositeStage) { + stageCount++; + + Stage locStage = stagesByTime.poll(); + + timings.add(locStage.toString() + " (parent=" + stage.description() + ")"); + } + } + } + + return timings; + } + finally { + lock.readLock().unlock(); + } + } + + /** + * + */ + private class CompositeStage extends Stage { + /** Local stages. */ + private final Map> localStages; + + /** + * @param description Description. + * @param time Time. + * @param localStages Local stages. + */ + public CompositeStage(String description, long time, Map> localStages) { + super(description, time); + + this.localStages = localStages; + } + + /** + * + */ + public Map> localStages() { + return localStages; + } + } + + /** + * + */ + private class Stage implements Comparable { + /** Description. */ + private final String description; + + /** Time. */ + private final long time; + + /** + * @param description Description. + * @param time Time. + */ + public Stage(String description, long time) { + this.description = description; + this.time = time; + } + + /** + * + */ + public String description() { + return description; + } + + /** + * + */ + public long time() { + return time; + } + + /** {@inheritDoc} */ + @Override public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("stage=").append('"').append(description()).append('"'); + sb.append(' ').append('(').append(time()).append(' ').append(measurementUnitShort()).append(')'); + + return sb.toString(); + } + + /** {@inheritDoc} */ + @Override public int compareTo(@NotNull TimeBag.Stage o) { + if (o.time > time) + return -1; + if (o.time < time) + return 1; + return o.description.compareTo(description); + } + } +} From 4f584ecb01f5ff3665c15ec7e0617b750be0e963 Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Fri, 21 Dec 2018 18:09:49 +0300 Subject: [PATCH 264/403] IGNITE-9303 Fixed rotated page ID and physical recovery after crash during recovery - Fixes #5580. Signed-off-by: Alexey Goncharuk --- .../wal/IgniteWriteAheadLogManager.java | 10 + .../cache/persistence/DataStructure.java | 2 +- .../GridCacheDatabaseSharedManager.java | 35 +- .../persistence/pagemem/PageMemoryImpl.java | 14 + .../wal/FileWriteAheadLogManager.java | 12 + .../FsyncModeFileWriteAheadLogManager.java | 12 + .../serializer/RecordDataV1Serializer.java | 2 +- .../serializer/RecordDataV2Serializer.java | 4 +- .../internal/util/OffheapReadWriteLock.java | 2 +- .../db/CheckpointFailingIoFactory.java | 88 +++++ .../db/IgniteLogicalRecoveryTest.java | 36 +- ...IgniteSequentialNodeCrashRecoveryTest.java | 337 ++++++++++++++++++ .../persistence/pagemem/NoOpWALManager.java | 5 + .../wal/memtracker/PageMemoryTracker.java | 12 +- .../IgnitePdsWithIndexingCoreTestSuite.java | 3 + 15 files changed, 519 insertions(+), 55 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointFailingIoFactory.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteSequentialNodeCrashRecoveryTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java index 8a4d3a706658e..7b8333f17aa04 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java @@ -93,6 +93,16 @@ public WALPointer log(WALRecord entry, RolloverType rolloverType) */ public void flush(WALPointer ptr, boolean explicitFsync) throws IgniteCheckedException, StorageException; + /** + * Reads WAL record by the specified pointer. + * + * @param ptr WAL pointer. + * @return WAL record. + * @throws IgniteCheckedException If failed to read. + * @throws StorageException If IO error occurred while reading WAL entries. + */ + public WALRecord read(WALPointer ptr) throws IgniteCheckedException, StorageException; + /** * Invoke this method to iterate over the written log entries. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java index c23a9700046c6..610f6e638d0db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java @@ -360,7 +360,7 @@ protected final long recyclePage( boolean needWalDeltaRecord = needWalDeltaRecord(pageId, page, walPlc); - if (PageIdUtils.tag(pageId) == FLAG_DATA) { + if (PageIdUtils.flag(pageId) == FLAG_DATA) { int rotatedIdPart = PageIO.getRotatedIdPart(pageAddr); if (rotatedIdPart != 0) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 336aa427e4320..0221cb38dec7d 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -94,6 +94,7 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; +import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord; import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.MvccDataEntry; import org.apache.ignite.internal.pagemem.wal.record.MvccTxRecord; @@ -366,6 +367,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan private final boolean recoveryVerboseLogging = IgniteSystemProperties.getBoolean( IgniteSystemProperties.IGNITE_RECOVERY_VERBOSE_LOGGING, true); + /** Pointer to a memory recovery record that should be included into the next checkpoint record. */ + private volatile WALPointer memoryRecoveryRecordPtr; + /** * @param ctx Kernal context. */ @@ -946,6 +950,10 @@ else if (restored != null) // Wal logging is now available. cctx.wal().resumeLogging(restored); + // Log MemoryRecoveryRecord to make sure that old physical records are not replayed during + // next physical recovery. + memoryRecoveryRecordPtr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis())); + for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext())) lsnr.afterBinaryMemoryRestore(this, binaryState); @@ -2021,6 +2029,8 @@ private RestoreBinaryState performBinaryMemoryRestore( log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']'); + WALPointer recPtr = status.endPtr; + boolean apply = status.needRestoreMemory(); if (apply) { @@ -2029,6 +2039,21 @@ private RestoreBinaryState performBinaryMemoryRestore( "finish checkpoint on node start."); cctx.pageStore().beginRecover(); + + WALRecord rec = cctx.wal().read(status.startPtr); + + if (!(rec instanceof CheckpointRecord)) + throw new StorageException("Checkpoint marker doesn't point to checkpoint record " + + "[ptr=" + status.startPtr + ", rec=" + rec + "]"); + + WALPointer cpMark = ((CheckpointRecord)rec).checkpointMark(); + + if (cpMark != null) { + log.info("Restoring checkpoint after logical recovery, will start physical recovery from " + + "back pointer: " + cpMark); + + recPtr = cpMark; + } } else cctx.wal().notchLastCheckpointPtr(status.startPtr); @@ -2037,7 +2062,7 @@ private RestoreBinaryState performBinaryMemoryRestore( long lastArchivedSegment = cctx.wal().lastArchivedSegment(); - WALIterator it = cctx.wal().replay(status.endPtr, recordTypePredicate); + WALIterator it = cctx.wal().replay(recPtr, recordTypePredicate); RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, it, lastArchivedSegment, cacheGroupsPredicate); @@ -2069,7 +2094,7 @@ private RestoreBinaryState performBinaryMemoryRestore( long page = pageMem.acquirePage(grpId, pageId, true); try { - long pageAddr = pageMem.writeLock(grpId, pageId, page); + long pageAddr = pageMem.writeLock(grpId, pageId, page, true); try { PageUtils.putBytes(pageAddr, 0, pageRec.pageData()); @@ -2141,7 +2166,7 @@ private RestoreBinaryState performBinaryMemoryRestore( long page = pageMem.acquirePage(grpId, pageId, true); try { - long pageAddr = pageMem.writeLock(grpId, pageId, page); + long pageAddr = pageMem.writeLock(grpId, pageId, page, true); try { r.applyDelta(pageMem, pageAddr); @@ -3501,7 +3526,9 @@ private void waitCheckpointEvent() { */ @SuppressWarnings("TooBroadScope") private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws IgniteCheckedException { - CheckpointRecord cpRec = new CheckpointRecord(null); + CheckpointRecord cpRec = new CheckpointRecord(memoryRecoveryRecordPtr); + + memoryRecoveryRecordPtr = null; WALPointer cpPtr = null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index c5cdfcee5bb4b..272d13f4fd992 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -1874,6 +1874,20 @@ private int pages() { } } + /** + * Gets a collection of all pages currently marked as dirty. Will create a collection copy. + * + * @return Collection of all page IDs marked as dirty. + */ + public Collection dirtyPages() { + Collection res = new HashSet<>((int)loadedPages()); + + for (Segment seg : segments) + res.addAll(seg.dirtyPages); + + return res; + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 819317ea08264..85139a750dfef 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -933,6 +933,18 @@ private FileWriteHandle closeBufAndRollover( cur.fsync(filePtr); } + /** {@inheritDoc} */ + @Override public WALRecord read(WALPointer ptr) throws IgniteCheckedException, StorageException { + try (WALIterator it = replay(ptr)) { + IgniteBiTuple rec = it.next(); + + if (rec.get1().equals(ptr)) + return rec.get2(); + else + throw new StorageException("Failed to read record by pointer [ptr=" + ptr + ", rec=" + rec + "]"); + } + } + /** {@inheritDoc} */ @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException { return replay(start, null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index c961f5201e2a0..098f34ba344b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -800,6 +800,18 @@ else if (rolloverType == RolloverType.CURRENT_SEGMENT) { cur.fsync(filePtr, false); } + /** {@inheritDoc} */ + @Override public WALRecord read(WALPointer ptr) throws IgniteCheckedException, StorageException { + try (WALIterator it = replay(ptr)) { + IgniteBiTuple rec = it.next(); + + if (rec.get1().equals(ptr)) + return rec.get2(); + else + throw new StorageException("Failed to read record by pointer [ptr=" + ptr + ", rec=" + rec + "]"); + } + } + /** {@inheritDoc} */ @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException { return replay(start, null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index 46f36119fb83b..f1b9a8f2c88a5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -558,7 +558,7 @@ WALRecord readPlainRecord(RecordType type, ByteBufferBackedDataInput in, long msb = in.readLong(); long lsb = in.readLong(); boolean hasPtr = in.readByte() != 0; - int idx = hasPtr ? in.readInt() : 0; + long idx = hasPtr ? in.readLong() : 0; int off = hasPtr ? in.readInt() : 0; int len = hasPtr ? in.readInt() : 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java index dd25b8cca4049..aaf3cb77720e9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java @@ -55,7 +55,7 @@ /** * Record data V2 serializer. */ -public class RecordDataV2Serializer extends RecordDataV1Serializer implements RecordDataSerializer { +public class RecordDataV2Serializer extends RecordDataV1Serializer { /** Length of HEADER record data. */ private static final int HEADER_RECORD_DATA_SIZE = /*Magic*/8 + /*Version*/4; @@ -125,7 +125,7 @@ public RecordDataV2Serializer(GridCacheSharedContext cctx) { long msb = in.readLong(); long lsb = in.readLong(); boolean hasPtr = in.readByte() != 0; - int idx0 = hasPtr ? in.readInt() : 0; + long idx0 = hasPtr ? in.readLong() : 0; int off = hasPtr ? in.readInt() : 0; int len = hasPtr ? in.readInt() : 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java index b119960cfc822..95c007be7ad96 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/OffheapReadWriteLock.java @@ -263,7 +263,7 @@ public void writeUnlock(long lock, int tag) { if (lockCount(state) != -1) throw new IllegalMonitorStateException("Attempted to release write lock while not holding it " + - "[lock=" + U.hexLong(lock) + ", state=" + U.hexLong(state)); + "[lock=" + U.hexLong(lock) + ", state=" + U.hexLong(state) + ']'); updated = releaseWithTag(state, tag); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointFailingIoFactory.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointFailingIoFactory.java new file mode 100644 index 0000000000000..2d48cd614b449 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointFailingIoFactory.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.OpenOption; + +/** + * + */ +class CheckpointFailingIoFactory implements FileIOFactory { + /** */ + private volatile boolean fail; + + /** + * Will fail immediately. + */ + public CheckpointFailingIoFactory() { + this(true); + } + + /** + * @param failImmediately Fail immediately flag. + */ + public CheckpointFailingIoFactory(boolean failImmediately) { + fail = failImmediately; + } + + /** + * After this call all subsequent write calls will fail. + */ + public void startFailing() { + fail = true; + } + + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... modes) throws IOException { + FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); + + if (file.getName().contains("part-")) + return new FileIODecorator(delegate) { + @Override public int write(ByteBuffer srcBuf) throws IOException { + if (fail) + throw new IOException("test"); + else + return super.write(srcBuf); + } + + @Override public int write(ByteBuffer srcBuf, long position) throws IOException { + if (fail) + throw new IOException("test"); + else + return super.write(srcBuf, position); + } + + @Override public int write(byte[] buf, int off, int len) throws IOException { + if (fail) + throw new IOException("test"); + else + return super.write(buf, off, len); + } + }; + + return delegate; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java index 4a4c82193e518..11bd791a99197 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java @@ -17,10 +17,6 @@ package org.apache.ignite.internal.processors.cache.persistence.db; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.OpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -56,10 +52,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheUtils; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; -import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; -import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; -import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -359,7 +352,7 @@ public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception { stopGrid(2, false); - ioFactory = new CheckpointFailIoFactory(); + ioFactory = new CheckpointFailingIoFactory(); IgniteInternalFuture startNodeFut = GridTestUtils.runAsync(() -> startGrid(2)); @@ -625,33 +618,6 @@ public void consistencyCheck(IgniteEx ignite) { } } - /** - * - */ - static class CheckpointFailIoFactory implements FileIOFactory { - /** {@inheritDoc} */ - @Override public FileIO create(File file, OpenOption... modes) throws IOException { - FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); - - if (file.getName().contains("part-")) - return new FileIODecorator(delegate) { - @Override public int write(ByteBuffer srcBuf) throws IOException { - throw new IOException("test"); - } - - @Override public int write(ByteBuffer srcBuf, long position) throws IOException { - throw new IOException("test"); - } - - @Override public int write(byte[] buf, int off, int len) throws IOException { - throw new IOException("test"); - } - }; - - return delegate; - } - } - /** * Test payload with indexed field. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteSequentialNodeCrashRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteSequentialNodeCrashRecoveryTest.java new file mode 100644 index 0000000000000..9135ba7376666 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteSequentialNodeCrashRecoveryTest.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.OpenOption; +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureHandler; +import org.apache.ignite.failure.StopNodeFailureHandler; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * + */ +public class IgniteSequentialNodeCrashRecoveryTest extends GridCommonAbstractTest { + /** */ + private static final int PAGE_SIZE = 4096; + + /** */ + private FileIOFactory fileIoFactory; + + /** */ + private FailureHandler failureHnd; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setMaxSize(512 * 1024 * 1024).setPersistenceEnabled(true)) + // Set large checkpoint frequency to make sure no checkpoint happens right after the node start. + .setCheckpointFrequency(getTestTimeout()) + .setPageSize(PAGE_SIZE); + + if (fileIoFactory != null) + dsCfg.setFileIOFactory(fileIoFactory); + + cfg + .setDataStorageConfiguration(dsCfg) + .setConsistentId(igniteInstanceName); + + if (failureHnd != null) + cfg.setFailureHandler(failureHnd); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** + * @throws Exception if failed. + */ + public void testCrashOnCheckpointAfterLogicalRecovery() throws Exception { + IgniteEx g = startGrid(0); + + g.cluster().active(true); + + g.getOrCreateCache(new CacheConfiguration<>("cache") + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setAffinity(new RendezvousAffinityFunction(false, 8))); + + disableCheckpoints(g); + + { + IgniteCache cache = g.cache("cache"); + + // Now that checkpoints are disabled, put some data to the cache. + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < 400; i++) + cache.put(i % 100, Thread.currentThread().getName()); + }, 64, "update-thread"); + } + + Collection dirtyAfterLoad = captureDirtyPages(g); + + stopGrid(0); + + CheckpointFailingIoFactory f = (CheckpointFailingIoFactory)(fileIoFactory = new CheckpointFailingIoFactory(false)); + StopLatchFailureHandler fh = (StopLatchFailureHandler)(failureHnd = new StopLatchFailureHandler()); + + // Now start the node. Since the checkpoint was disabled, logical recovery will be performed. + g = startGrid(0); + + fileIoFactory = null; + failureHnd = null; + + // Capture dirty pages after logical recovery & updates. + Collection dirtyAfterRecoveryAndUpdates = captureDirtyPages(g); + + f.startFailing(); + + triggerCheckpoint(g); + + assertTrue("Failed to wait for checkpoint failure", fh.waitFailed()); + + // Capture pages we marked on first run and did not mark on second run. + dirtyAfterLoad.removeAll(dirtyAfterRecoveryAndUpdates); + + assertFalse(dirtyAfterLoad.isEmpty()); + + fileIoFactory = new CheckingIoFactory(dirtyAfterLoad); + + g = startGrid(0); + + { + IgniteCache cache = g.cache("cache"); + + for (int i = 0; i < 400; i++) + cache.put(100 + (i % 100), Thread.currentThread().getName()); + + for (int i = 0; i < 200; i++) + assertTrue("i=" + i, cache.containsKey(i)); + } + } + + /** + * + */ + private void disableCheckpoints(IgniteEx g) throws Exception { + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)g.context() + .cache().context().database(); + + dbMgr.enableCheckpoints(false).get(); + } + + /** + * @param ig Ignite instance. + */ + private void triggerCheckpoint(IgniteEx ig) { + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig.context() + .cache().context().database(); + + dbMgr.wakeupForCheckpoint("test-should-fail"); + } + + /** + * @param g Ignite instance. + * @throws IgniteCheckedException If failed. + */ + private Collection captureDirtyPages(IgniteEx g) throws IgniteCheckedException { + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)g.context() + .cache().context().database(); + + // Capture a set of dirty pages. + PageMemoryImpl pageMem = (PageMemoryImpl)dbMgr.dataRegion("default").pageMemory(); + + return pageMem.dirtyPages(); + } + + /** + * + */ + private class StopLatchFailureHandler extends StopNodeFailureHandler { + /** */ + private CountDownLatch stopLatch = new CountDownLatch(1); + + /** {@inheritDoc} */ + @Override public boolean handle(Ignite ignite, FailureContext failureCtx) { + new Thread( + new Runnable() { + @Override public void run() { + U.error(ignite.log(), "Stopping local node on Ignite failure: [failureCtx=" + failureCtx + ']'); + + IgnitionEx.stop(ignite.name(), true, true); + + stopLatch.countDown(); + } + }, + "node-stopper" + ).start(); + + return true; + } + + /** + * @return {@code true} if wait succeeded. + * @throws InterruptedException If current thread was interrupted. + */ + public boolean waitFailed() throws InterruptedException { + return stopLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(StopNodeFailureHandler.class, this, "super", super.toString()); + } + } + + /** + * + */ + private static class CheckingIoFactory implements FileIOFactory { + /** */ + private final transient Collection forbiddenPages; + + /** + * @param forbiddenPages Forbidden pages. + */ + private CheckingIoFactory(Collection forbiddenPages) { + this.forbiddenPages = forbiddenPages; + } + + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... modes) throws IOException { + FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); + + if (file.getName().contains("part-")) + return new CheckingFileIO(file, delegate, forbiddenPages); + + return delegate; + } + } + + /** + * + */ + private static class CheckingFileIO extends FileIODecorator { + /** */ + private int grpId; + + /** */ + private int partId; + + /** */ + private Collection forbiddenPages; + + /** + * @param file File. + * @param delegate Delegate. + * @param forbiddenPages Forbidden pages. + */ + public CheckingFileIO(File file, FileIO delegate, Collection forbiddenPages) { + super(delegate); + this.forbiddenPages = forbiddenPages; + + String fileName = file.getName(); + + int start = fileName.indexOf("part-") + 5; + int end = fileName.indexOf(".bin"); + partId = Integer.parseInt(fileName.substring(start, end)); + + String path = file.getPath(); + + if (path.contains(File.separator + "metastorage" + File.separator)) + grpId = MetaStorage.METASTORAGE_CACHE_ID; + else { + start = path.indexOf("cache-") + 6; + end = path.indexOf(File.separator, start); + + grpId = start >= 0 ? CU.cacheId(path.substring(start, end)) : 0; + } + } + + /** {@inheritDoc} */ + @Override public int write(ByteBuffer srcBuf) throws IOException { + throw new AssertionError("Should not be called"); + } + + /** {@inheritDoc} */ + @Override public int write(ByteBuffer srcBuf, long position) throws IOException { + FullPageId fId = new FullPageId( + PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, (int)(position / PAGE_SIZE) - 1), + grpId); + + if (forbiddenPages.contains(fId)) + throw new AssertionError("Attempted to write invalid page on recovery: " + fId); + + return super.write(srcBuf, position); + } + + /** {@inheritDoc} */ + @Override public int write(byte[] buf, int off, int len) throws IOException { + throw new AssertionError("Should not be called"); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java index 8d854bb32117a..b482c84075e66 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java @@ -69,6 +69,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { } + /** {@inheritDoc} */ + @Override public WALRecord read(WALPointer ptr) throws IgniteCheckedException, StorageException { + return null; + } + /** {@inheritDoc} */ @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException { return null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java index 864d2cc0137e8..9ab807e334ee6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java @@ -47,9 +47,7 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; @@ -434,8 +432,6 @@ private void applyWalRecord(WALRecord record) throws IgniteCheckedException { try { PageUtils.putBytes(page.address(), 0, snapshot.pageData()); - page.fullPageId(fullPageId); - page.changeHistory().clear(); page.changeHistory().add(record); @@ -459,12 +455,6 @@ else if (record instanceof PageDeltaRecord) { try { deltaRecord.applyDelta(pageMemoryMock, page.address()); - // Set new fullPageId after recycle or after new page init, because pageId tag is changed. - if (record instanceof RecycleRecord) - page.fullPageId(new FullPageId(((RecycleRecord)record).newPageId(), grpId)); - else if (record instanceof InitNewPageRecord) - page.fullPageId(new FullPageId(((InitNewPageRecord)record).newPageId(), grpId)); - page.changeHistory().add(record); } finally { @@ -565,7 +555,7 @@ public boolean checkPages(boolean checkAll) throws IgniteCheckedException { long rmtPage = pageMem.acquirePage(fullPageId.groupId(), fullPageId.pageId()); try { - long rmtPageAddr = pageMem.readLock(fullPageId.groupId(), fullPageId.pageId(), rmtPage); + long rmtPageAddr = pageMem.readLockForce(fullPageId.groupId(), fullPageId.pageId(), rmtPage); try { page.lock(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java index 214a541a17214..cdb4ee8aacd9e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgniteLogicalRecoveryTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgniteSequentialNodeCrashRecoveryTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheDestroyDuringCheckpointTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheIntegrationTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsDiskErrorsRecoveringTest; @@ -91,6 +92,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteLogicalRecoveryTest.class); + suite.addTestSuite(IgniteSequentialNodeCrashRecoveryTest.class); + return suite; } } From 328cb1d8d78e39cf1ed12d1b9acc118952cc7686 Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Fri, 21 Dec 2018 18:31:38 +0300 Subject: [PATCH 265/403] IGNITE-10658 Visor: Fixed rebalance progress calculation logic. Fixes #5706. (cherry picked from commit 59c5420345a0decadced3d3b9b1e86ce0f7dda47) --- .../VisorCacheRebalanceCollectorTask.java | 2 +- .../visor/node/VisorNodeDataCollectorJob.java | 4 +- .../CacheGroupsMetricsRebalanceTest.java | 74 +++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java index eda9f9419c100..9d0e04144ea58 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java @@ -158,7 +158,7 @@ private VisorCacheRebalanceCollectorJob(VisorCacheRebalanceCollectorTaskArg arg, else if (total == 0 && rebalanceInProgress) res.setRebalance(MINIMAL_REBALANCE); else - res.setRebalance(total > 0 ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE); + res.setRebalance(total > 0 && rebalanceInProgress ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE); } catch (Exception e) { res.setRebalance(REBALANCE_NOT_AVAILABLE); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index 9a7d2b146618e..3a115dffd5a1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -249,7 +249,9 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto else if (total == 0 && rebalanceInProgress) res.setRebalance(MINIMAL_REBALANCE); else - res.setRebalance(total > 0 ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE); + res.setRebalance(total > 0 && rebalanceInProgress + ? Math.max(ready / total, MINIMAL_REBALANCE) + : REBALANCE_COMPLETE); } catch (Exception e) { res.setRebalance(REBALANCE_NOT_AVAILABLE); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java index af2dc633d825e..3e90625cf2cd3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.cache; +import java.util.Collections; +import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.ignite.Ignite; @@ -31,13 +33,19 @@ import org.apache.ignite.events.CacheRebalancingEvent; import org.apache.ignite.events.Event; import org.apache.ignite.events.EventType; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTask; +import org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTaskArg; +import org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTaskResult; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteSystemProperties.IGNITE_REBALANCE_STATISTICS_TIME_INTERVAL; @@ -168,6 +176,72 @@ public void testRebalance() throws Exception { assertTrue(ratio > 0.9 && ratio < 1.1); } + /** + * @throws Exception If failed. + */ + public void testRebalanceProgressUnderLoad() throws Exception { + Ignite ignite = startGrids(4); + + IgniteCache cache1 = ignite.cache(CACHE1); + + Random r = new Random(); + + GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + for (int i = 0; i < 100_000; i++) { + int next = r.nextInt(); + + cache1.put(next, CACHE1 + "-" + next); + } + } + }); + + IgniteEx ig = startGrid(4); + + GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + for (int i = 0; i < 100_000; i++) { + int next = r.nextInt(); + + cache1.put(next, CACHE1 + "-" + next); + } + } + }); + + CountDownLatch latch = new CountDownLatch(1); + + ig.events().localListen(new IgnitePredicate() { + @Override public boolean apply(Event evt) { + latch.countDown(); + + return false; + } + }, EventType.EVT_CACHE_REBALANCE_STOPPED); + + latch.await(); + + VisorNodeDataCollectorTaskArg taskArg = new VisorNodeDataCollectorTaskArg(); + taskArg.setCacheGroups(Collections.emptySet()); + + VisorTaskArgument arg = new VisorTaskArgument<>( + Collections.singletonList(ignite.cluster().localNode().id()), + taskArg, + false + ); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + VisorNodeDataCollectorTaskResult res = ignite.compute().execute(VisorNodeDataCollectorTask.class, arg); + + CacheMetrics snapshot = ig.cache(CACHE1).metrics(); + + return snapshot.getRebalancedKeys() > snapshot.getEstimatedRebalancingKeys() + && res.getRebalance().get(ignite.cluster().localNode().id()) == 1.0 + && snapshot.getRebalancingPartitionsCount() == 0; + } + }, 5000); + } + /** * @throws Exception If failed. */ From a1aff92ab01a924905cb1d4716832294717dee90 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Mon, 24 Dec 2018 15:11:22 +0700 Subject: [PATCH 266/403] IGNITE-10279 Control.sh: Unified options naming format. Fixes #5672. (cherry picked from commit 7d933e8e13c48a16b8184e0df4c37f71b7d57f33) --- .../internal/TransactionsMXBeanImpl.java | 8 +- .../ignite/internal/commandline/Command.java | 5 + .../internal/commandline/CommandHandler.java | 241 +++++++----- .../commandline/cache/CacheCommand.java | 5 + .../VerifyBackupPartitionsDumpTask.java | 5 +- .../internal/visor/tx/VisorTxSortOrder.java | 17 - .../CommandHandlerParsingTest.java | 12 +- .../ignite/util/GridCommandHandlerTest.java | 360 +++++++++--------- .../util/GridCommandHandlerIndexingTest.java | 6 +- 9 files changed, 356 insertions(+), 303 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java index 16738de12343d..a8a3c886617b8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java @@ -72,12 +72,8 @@ else if ("servers".equals(prj)) VisorTxSortOrder sortOrder = null; - if (order != null) { - if ("DURATION".equals(order)) - sortOrder = VisorTxSortOrder.DURATION; - else if ("SIZE".equals(order)) - sortOrder = VisorTxSortOrder.SIZE; - } + if (order != null) + sortOrder = VisorTxSortOrder.valueOf(order.toUpperCase()); VisorTxTaskArg arg = new VisorTxTaskArg(kill ? VisorTxOperation.KILL : VisorTxOperation.LIST, limit, minDuration == null ? null : minDuration * 1000, minSize, null, proj, consIds, xid, lbRegex, sortOrder); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java index c64e488db4ffd..1f7c0a34c57b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java @@ -71,4 +71,9 @@ public static Command of(String text) { public String text() { return text; } + + /** {@inheritDoc} */ + @Override public String toString() { + return text; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 450affa29b912..cd9693132d672 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -186,19 +186,19 @@ public class CommandHandler { private static final String CMD_AUTO_CONFIRMATION = "--yes"; /** */ - protected static final String CMD_PING_INTERVAL = "--ping-interval"; + private static final String CMD_PING_INTERVAL = "--ping-interval"; /** */ - protected static final String CMD_PING_TIMEOUT = "--ping-timeout"; + private static final String CMD_PING_TIMEOUT = "--ping-timeout"; /** */ private static final String CMD_DUMP = "--dump"; /** */ - private static final String CMD_SKIP_ZEROS = "--skipZeros"; + private static final String CMD_SKIP_ZEROS = "--skip-zeros"; /** Cache filter. */ - private static final String CACHE_FILTER = "--cacheFilter"; + private static final String CACHE_FILTER = "--cache-filter"; /** */ private static final String CMD_USER_ATTRIBUTES = "--user-attributes"; @@ -209,10 +209,10 @@ public class CommandHandler { private static final String CMD_SSL_PROTOCOL = "--ssl-protocol"; /** */ - private static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites"; + private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm"; /** */ - private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm"; + private static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites"; /** */ private static final String CMD_KEYSTORE = "--keystore"; @@ -250,16 +250,16 @@ public class CommandHandler { AUX_COMMANDS.add(CMD_PING_TIMEOUT); AUX_COMMANDS.add(CMD_SSL_PROTOCOL); + AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM); AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES); - AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM); - AUX_COMMANDS.add(CMD_KEYSTORE_TYPE); AUX_COMMANDS.add(CMD_KEYSTORE); AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD); + AUX_COMMANDS.add(CMD_KEYSTORE_TYPE); - AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE); AUX_COMMANDS.add(CMD_TRUSTSTORE); AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD); + AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE); } /** Broadcast uuid. */ @@ -284,10 +284,10 @@ public class CommandHandler { private static final String BASELINE_SET_VERSION = "version"; /** Parameter name for validate_indexes command. */ - static final String VI_CHECK_FIRST = "checkFirst"; + static final String VI_CHECK_FIRST = "--check-first"; /** Parameter name for validate_indexes command. */ - static final String VI_CHECK_THROUGH = "checkThrough"; + static final String VI_CHECK_THROUGH = "--check-through"; /** */ static final String WAL_PRINT = "print"; @@ -326,37 +326,34 @@ public class CommandHandler { private static final String VALIDATE_INDEXES_TASK = "org.apache.ignite.internal.visor.verify.VisorValidateIndexesTask"; /** */ - private static final String TX_LIMIT = "limit"; - - /** */ - private static final String TX_ORDER = "order"; + private static final String TX_LIMIT = "--limit"; /** */ - public static final String CMD_TX_ORDER_START_TIME = "START_TIME"; + private static final String TX_ORDER = "--order"; /** */ - private static final String TX_SERVERS = "servers"; + private static final String TX_SERVERS = "--servers"; /** */ - private static final String TX_CLIENTS = "clients"; + private static final String TX_CLIENTS = "--clients"; /** */ - private static final String TX_DURATION = "minDuration"; + private static final String TX_DURATION = "--min-duration"; /** */ - private static final String TX_SIZE = "minSize"; + private static final String TX_SIZE = "--min-size"; /** */ - private static final String TX_LABEL = "label"; + private static final String TX_LABEL = "--label"; /** */ - private static final String TX_NODES = "nodes"; + private static final String TX_NODES = "--nodes"; /** */ - private static final String TX_XID = "xid"; + private static final String TX_XID = "--xid"; /** */ - private static final String TX_KILL = "kill"; + private static final String TX_KILL = "--kill"; /** */ private static final String OUTPUT_FORMAT = "--output-format"; @@ -368,10 +365,10 @@ public class CommandHandler { private static final String UTILITY_NAME = "control.sh"; /** Common options. */ - private static final String COMMON_OPTIONS = String.join(" ", getCommonOptions()); + private static final String COMMON_OPTIONS = j(" ", getCommonOptions()); /** Utility name with common options. */ - private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS); + private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = j(" ", UTILITY_NAME, COMMON_OPTIONS); /** Indent for help output. */ private static final String INDENT = " "; @@ -379,6 +376,12 @@ public class CommandHandler { /** */ private static final String NULL = "null"; + /** */ + private static final String NODE_ID = "nodeId"; + + /** */ + private static final String OP_NODE_ID = op(NODE_ID); + /** */ private Iterator argsIt; @@ -409,14 +412,14 @@ private static List getCommonOptions() { list.add(op(CMD_PING_INTERVAL, "PING_INTERVAL")); list.add(op(CMD_PING_TIMEOUT, "PING_TIMEOUT")); - list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]")); - list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]")); + list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ..., SSL_PROTOCOL_N]")); + list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ..., SSL_CIPHER_N]")); list.add(op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM")); list.add(op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE")); - list.add(op(CMD_KEYSTORE, "KEYSTORE")); + list.add(op(CMD_KEYSTORE, "KEYSTORE_PATH")); list.add(op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD")); list.add(op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE")); - list.add(op(CMD_TRUSTSTORE, "TRUSTSTORE")); + list.add(op(CMD_TRUSTSTORE, "TRUSTSTORE_PATH")); list.add(op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD")); return list; @@ -432,25 +435,27 @@ private void log(String s) { } /** - * Adds indent to begin of input string. + * Adds indent to begin of object's string representation. * - * @param s Input string. + * @param o Input object. * @return Indented string. */ - private static String i(String s) { - return i(s, 1); + private static String i(Object o) { + return i(o, 1); } /** - * Adds specified indents to begin of input string. + * Adds specified indents to begin of object's string representation. * - * @param s Input string. + * @param o Input object. * @param indentCnt Number of indents. * @return Indented string. */ - private static String i(String s, int indentCnt) { + private static String i(Object o, int indentCnt) { assert indentCnt >= 0; + String s = o == null ? null : o.toString(); + switch (indentCnt) { case 0: return s; @@ -459,14 +464,15 @@ private static String i(String s, int indentCnt) { return INDENT + s; default: - SB sb = new SB(s.length() + indentCnt * INDENT.length()); + int sLen = s == null ? 4 : s.length(); + + SB sb = new SB(sLen + indentCnt * INDENT.length()); for (int i = 0; i < indentCnt; i++) sb.a(INDENT); return sb.a(s).toString(); } - } /** @@ -689,12 +695,15 @@ private Stream>> listHostsByClientNod GridClient client ) throws GridClientException { return client.compute().nodes(GridClientNode::connectable).stream() - .map(node -> new IgniteBiTuple<>(node, - Stream.concat( - node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(), - node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream() + .map( + node -> new IgniteBiTuple<>( + node, + Stream.concat( + node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(), + node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream() + ) + .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList()) ) - .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())) ); } @@ -838,28 +847,22 @@ private void cache(GridClient client, CacheArguments cacheArgs) throws Throwable } } - /** - * - */ + /** */ private void printCacheHelp() { - log(i("The '" + CACHE.text() + " subcommand' is used to get information about and perform actions with caches. The command has the following syntax:")); + log(i("The '" + CACHE + " subcommand' is used to get information about and perform actions with caches. The command has the following syntax:")); nl(); - log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + CACHE.text() + "[subcommand] ")); + log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + CACHE + "[subcommand] ")); nl(); - log(i("The subcommands that take [nodeId] as an argument ('" + LIST.text() + "', '" + CONTENTION.text() + "' and '" + VALIDATE_INDEXES.text() + "') will be executed on the given node or on all server nodes if the option is not specified. Other commands will run on a random server node.")); + log(i("The subcommands that take " + OP_NODE_ID + " as an argument ('" + LIST + "', '" + CONTENTION + "' and '" + VALIDATE_INDEXES + "') will be executed on the given node or on all server nodes if the option is not specified. Other commands will run on a random server node.")); nl(); nl(); log(i("Subcommands:")); - usageCache(LIST, "regexPattern", op(or("groups","seq")), op("nodeId"), op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE - .text())); - usageCache(CONTENTION, "minQueueSize", op("nodeId"), op("maxPrint")); - usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", - op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(), CacheFilterEnum.PERSISTENT.toString(), - CacheFilterEnum.NOT_PERSISTENT.toString()))); - usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", op("nodeId"), op(or(VI_CHECK_FIRST + " N", - VI_CHECK_THROUGH + " K"))); - usageCache(DISTRIBUTION, or("nodeId", NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attName1,...,attrNameN")); + usageCache(LIST, "regexPattern", op(or("groups", "seq")), OP_NODE_ID, op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE)); + usageCache(CONTENTION, "minQueueSize", OP_NODE_ID, op("maxPrint")); + usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", op(CACHE_FILTER, or(CacheFilterEnum.values()))); + usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", OP_NODE_ID, op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K"))); + usageCache(DISTRIBUTION, or(NODE_ID, NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attrName1,...,attrNameN")); usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN"); nl(); } @@ -936,7 +939,7 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t if (!integrityCheckFailures.isEmpty()) { for (IndexIntegrityCheckIssue is : integrityCheckFailures) - log(i(is.toString())); + log(i(is)); } Map partRes = nodeEntry.getValue().partitionResult(); @@ -945,10 +948,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t ValidateIndexesPartitionResult res = e.getValue(); if (!res.issues().isEmpty()) { - log(i(e.getKey().toString() + " " + e.getValue().toString())); + log(i(j(" ", e.getKey(), e.getValue()))); for (IndexValidationIssue is : res.issues()) - log(i(is.toString(), 2)); + log(i(is, 2)); } } @@ -958,10 +961,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t ValidateIndexesPartitionResult res = e.getValue(); if (!res.issues().isEmpty()) { - log(i("SQL Index " + e.getKey() + " " + e.getValue().toString())); + log(i(j(" ", "SQL Index", e.getKey(), e.getValue()))); for (IndexValidationIssue is : res.issues()) - log(i(is.toString(),2)); + log(i(is, 2)); } } } @@ -1628,7 +1631,7 @@ private boolean isConnectionError(Throwable e) { */ private void usage(String desc, Command cmd, String... args) { log(desc); - log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + cmd.text() + " " + String.join(" ", args), 2)); + log(i(j(" ", UTILITY_NAME, cmd, j(" ", args)), 2)); nl(); } @@ -1652,7 +1655,7 @@ private void usageCache(CacheCommand cmd, String... args) { private void usageCache(int indentsNum, CacheCommand cmd, String... args) { log(i(DELIM, indentsNum)); nl(); - log(i(CACHE.text() + " " + cmd.text() + " " + String.join(" ", args), indentsNum++)); + log(i(j(" ", CACHE, cmd, j(" ", args)), indentsNum++)); nl(); log(i(getCacheSubcommandDesc(cmd), indentsNum)); nl(); @@ -1762,34 +1765,54 @@ private Map createCacheArgsDesc(CacheCommand cmd) { /** * Join input parameters with space and wrap optional braces {@code []}. * - * @param param First input parameter. * @param params Other input parameter. * @return Joined parameters wrapped optional braces. */ - private static String op(String param, String... params) { - if (params == null || params.length == 0) - return "[" + param + "]"; + private static String op(Object... params) { + return j(new SB(), "[", " ", params).a("]").toString(); + } - return "[" + param + " " + String.join(" ", params) + "]"; + /** + * Join input parameters with specified {@code delimeter} between them. + * + * @param delimeter Specified delimeter. + * @param params Other input parameter. + * @return Joined paramaters with specified {@code delimeter}. + */ + private static String j(String delimeter, Object... params) { + return j(new SB(), "", delimeter, params).toString(); } /** - * Concatenates input parameters to single string with OR delimiter {@code |}. + * Join input parameters with specified {@code delimeter} between them and append to the end {@code delimeter}. * - * @param param1 First parameter. - * @param params Remaining parameters. - * @return Concatenated string. + * @param sb Specified string builder. + * @param sbDelimeter Delimeter between {@code sb} and appended {@code param}. + * @param delimeter Specified delimeter. + * @param params Other input parameter. + * @return SB with appended to the end joined paramaters with specified {@code delimeter}. */ - private static String or(String param1, String... params) { - if (params.length == 0) - return param1; + private static SB j(SB sb, String sbDelimeter, String delimeter, Object... params) { + if (!F.isEmpty(params)) { + sb.a(sbDelimeter); - SB sb = new SB(param1); + for (Object par : params) + sb.a(par).a(delimeter); - for (String param : params) - sb.a("|").a(param); + sb.setLength(sb.length() - delimeter.length()); + } - return sb.toString(); + return sb; + } + + /** + * Concatenates input parameters to single string with OR delimiter {@code |}. + * + * @param params Remaining parameters. + * @return Concatenated string. + */ + private static String or(Object... params) { + return j("|", params); } /** @@ -2107,9 +2130,9 @@ else if (CMD_SKIP_ZEROS.equals(nextArg)) cacheArgs.skipZeros(true); else if (CACHE_FILTER.equals(nextArg)) { String filter = nextArg("The cache filter should be specified. The following values can be " + - "used: " + Arrays.toString(CacheFilterEnum.values()) + '.').toUpperCase(); + "used: " + Arrays.toString(CacheFilterEnum.values()) + '.'); - cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter)); + cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter.toUpperCase())); } else parseCacheNames(nextArg, cacheArgs); @@ -2352,7 +2375,7 @@ private VisorTxTaskArg parseTransactionArguments() { case TX_ORDER: nextArg(""); - sortOrder = VisorTxSortOrder.fromString(nextArg(TX_ORDER)); + sortOrder = VisorTxSortOrder.valueOf(nextArg(TX_ORDER).toUpperCase()); break; @@ -2605,27 +2628,56 @@ private List split(String s, String delim) { .collect(Collectors.toList()); } + /** + * @return Transaction command options. + */ + private String[] getTxOptions() { + List list = new ArrayList<>(); + + list.add(op(TX_XID, "XID")); + list.add(op(TX_DURATION, "SECONDS")); + list.add(op(TX_SIZE, "SIZE")); + list.add(op(TX_LABEL, "PATTERN_REGEX")); + list.add(op(or(TX_SERVERS, TX_CLIENTS))); + list.add(op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]")); + list.add(op(TX_LIMIT, "NUMBER")); + list.add(op(TX_ORDER, or(VisorTxSortOrder.values()))); + list.add(op(TX_KILL)); + list.add(op(CMD_AUTO_CONFIRMATION)); + + return list.toArray(new String[list.size()]); + } + /** */ private void printHelp() { + final String constistIds = "consistentId1[,consistentId2,....,consistentIdN]"; + + log("Control.sh is used to execute admin commands on cluster or get common cluster info. The command has the following syntax:"); + nl(); + + log(i(j(" ", UTILITY_NAME_WITH_COMMON_OPTIONS, op("command"), ""))); + nl(); + nl(); + log("This utility can do the following commands:"); usage(i("Activate cluster:"), ACTIVATE); usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION)); usage(i("Print current cluster state:"), STATE); usage(i("Print cluster baseline topology:"), BASELINE); - usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); - usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); - usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); + usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, constistIds, op(CMD_AUTO_CONFIRMATION)); + usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, constistIds, op(CMD_AUTO_CONFIRMATION)); + usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, constistIds, op(CMD_AUTO_CONFIRMATION)); usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION)); - usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION)); + usage(i("List or kill transactions:"), TX, getTxOptions()); if (enableExperimental) { usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]"); - usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION)); + usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION)); } log(i("View caches information in a cluster. For more details type:")); - log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2)); + log(i(j(" ", UTILITY_NAME, CACHE, HELP), 2)); nl(); log("By default commands affecting the cluster require interactive confirmation."); @@ -2637,10 +2689,11 @@ private void printHelp() { log(i("PORT=" + DFLT_PORT, 2)); log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2)); log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2)); - log(i("SSL_PROTOCOL=" + DFLT_SSL_PROTOCOL, 2)); + log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2)); log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2)); - log(i("KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); - log(i("TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + log(i("KEYSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + log(i("TRUSTSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2)); + nl(); log("Exit codes:"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java index af222a8bc0973..63a55d8c9f41d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java @@ -100,4 +100,9 @@ public String text() { @Nullable public static CacheCommand fromOrdinal(int ord) { return ord >= 0 && ord < VALS.length ? VALS[ord] : null; } + + /** {@inheritDoc} */ + @Override public String toString() { + return name; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java index 5945f4d0990b1..c0fd36ae5a6b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java @@ -60,7 +60,7 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter= 0 && ord < VALS.length ? VALS[ord] : null; } - - /** - * @param name Name. - */ - public static VisorTxSortOrder fromString(String name) { - if (DURATION.toString().equals(name)) - return DURATION; - - if (SIZE.toString().equals(name)) - return SIZE; - - if (CommandHandler.CMD_TX_ORDER_START_TIME.equals(name)) - return START_TIME; - - throw new IllegalArgumentException("Sort order is unknown: " + name); - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 203d6d06d0ba3..45b0aab55b07e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -325,7 +325,7 @@ public void testParseAutoConfirmationFlag() { break; } case TX: { - args = hnd.parseAndValidate(asList(cmd.text(), "xid", "xid1", "minDuration", "10", "kill", "--yes")); + args = hnd.parseAndValidate(asList(cmd.text(), "--xid", "xid1", "--min-duration", "10", "--kill", "--yes")); assertEquals(cmd, args.command()); assertEquals(DFLT_HOST, args.host()); @@ -461,8 +461,8 @@ public void testTransactionArguments() { catch (IllegalArgumentException ignored) { } - args = hnd.parseAndValidate(asList("--tx", "minDuration", "120", "minSize", "10", "limit", "100", "order", "SIZE", - "servers")); + args = hnd.parseAndValidate(asList("--tx", "--min-duration", "120", "--min-size", "10", "--limit", "100", "--order", "SIZE", + "--servers")); VisorTxTaskArg arg = args.transactionArguments(); @@ -472,8 +472,8 @@ public void testTransactionArguments() { assertEquals(VisorTxSortOrder.SIZE, arg.getSortOrder()); assertEquals(VisorTxProjection.SERVER, arg.getProjection()); - args = hnd.parseAndValidate(asList("--tx", "minDuration", "130", "minSize", "1", "limit", "60", "order", "DURATION", - "clients")); + args = hnd.parseAndValidate(asList("--tx", "--min-duration", "130", "--min-size", "1", "--limit", "60", "--order", "DURATION", + "--clients")); arg = args.transactionArguments(); @@ -483,7 +483,7 @@ public void testTransactionArguments() { assertEquals(VisorTxSortOrder.DURATION, arg.getSortOrder()); assertEquals(VisorTxProjection.CLIENT, arg.getProjection()); - args = hnd.parseAndValidate(asList("--tx", "nodes", "1,2,3")); + args = hnd.parseAndValidate(asList("--tx", "--nodes", "1,2,3")); arg = args.transactionArguments(); diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6bb999d00e235..e21ca692f186c 100644 --- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -62,8 +62,8 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.commandline.Command; import org.apache.ignite.internal.commandline.CommandHandler; -import org.apache.ignite.internal.commandline.OutputFormat; import org.apache.ignite.internal.commandline.cache.CacheCommand; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; @@ -90,11 +90,9 @@ import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; -import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.tx.VisorTxInfo; import org.apache.ignite.internal.visor.tx.VisorTxTaskResult; -import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; @@ -115,7 +113,8 @@ import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR; import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE; import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE; -import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREMIX; +import static org.apache.ignite.internal.commandline.cache.CacheCommand.HELP; +import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREFIX; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; @@ -166,11 +165,12 @@ protected File folder(String folder) throws IgniteCheckedException { cleanPersistenceDir(); - //delete idle-verify dump files. + // Delete idle-verify dump files. try (DirectoryStream files = newDirectoryStream( - Paths.get(U.defaultWorkDirectory()), - entry -> entry.toFile().getName().startsWith(IDLE_DUMP_FILE_PREMIX) - )) { + Paths.get(U.defaultWorkDirectory()), + entry -> entry.toFile().getName().startsWith(IDLE_DUMP_FILE_PREFIX) + ) + ) { for (Path path : files) delete(path); } @@ -253,13 +253,11 @@ protected int execute(String... args) { * @param args Arguments. * @return Result of execution */ - protected int execute(ArrayList args) { - // Add force to avoid interactive confirmation - args.add(CMD_AUTO_CONFIRMATION); - - SB sb = new SB(); - - args.forEach(arg -> sb.a(arg).a(" ")); + protected int execute(List args) { + if(!F.isEmpty(args) && !"--help".equalsIgnoreCase(args.get(0))) { + // Add force to avoid interactive confirmation. + args.add(CMD_AUTO_CONFIRMATION); + } return new CommandHandler().execute(args); } @@ -480,9 +478,7 @@ public void testActiveTransactions() throws Exception { // Basic test. validate(h, map -> { - ClusterNode node = grid(0).cluster().localNode(); - - VisorTxTaskResult res = map.get(node); + VisorTxTaskResult res = map.get(grid(0).cluster().localNode()); for (VisorTxInfo info : res.getInfos()) { if (info.getSize() == 100) { @@ -503,7 +499,7 @@ public void testActiveTransactions() throws Exception { for (Map.Entry entry : map.entrySet()) assertEquals(entry.getKey().equals(node) ? 1 : 0, entry.getValue().getInfos().size()); - }, "--tx", "label", "label1"); + }, "--tx", "--label", "label1"); // Test filter by label regex. validate(h, map -> { @@ -525,7 +521,7 @@ else if (entry.getKey().equals(node2)) { assertTrue(entry.getValue().getInfos().isEmpty()); } - }, "--tx", "label", "^label[0-9]"); + }, "--tx", "--label", "^label[0-9]"); // Test filter by empty label. validate(h, map -> { @@ -534,7 +530,7 @@ else if (entry.getKey().equals(node2)) { for (VisorTxInfo info : res.getInfos()) assertNull(info.getLabel()); - }, "--tx", "label", "null"); + }, "--tx", "--label", "null"); // test check minSize int minSize = 10; @@ -546,21 +542,21 @@ else if (entry.getKey().equals(node2)) { for (VisorTxInfo txInfo : res.getInfos()) assertTrue(txInfo.getSize() >= minSize); - }, "--tx", "minSize", Integer.toString(minSize)); + }, "--tx", "--min-size", Integer.toString(minSize)); // test order by size. validate(h, map -> { VisorTxTaskResult res = map.get(grid(0).localNode()); assertTrue(res.getInfos().get(0).getSize() >= res.getInfos().get(1).getSize()); - }, "--tx", "order", "SIZE"); + }, "--tx", "--order", "SIZE"); // test order by duration. validate(h, map -> { VisorTxTaskResult res = map.get(grid(0).localNode()); assertTrue(res.getInfos().get(0).getDuration() >= res.getInfos().get(1).getDuration()); - }, "--tx", "order", "DURATION"); + }, "--tx", "--order", "DURATION"); // test order by start_time. validate(h, map -> { @@ -568,7 +564,7 @@ else if (entry.getKey().equals(node2)) { for (int i = res.getInfos().size() - 1; i > 1; i--) assertTrue(res.getInfos().get(i - 1).getStartTime() >= res.getInfos().get(i).getStartTime()); - }, "--tx", "order", CommandHandler.CMD_TX_ORDER_START_TIME); + }, "--tx", "--order", "START_TIME"); // Trigger topology change and test connection. IgniteInternalFuture startFut = multithreadedAsync(() -> { @@ -593,9 +589,9 @@ else if (entry.getKey().equals(node2)) { VisorTxInfo info = killedEntry.getValue().getInfos().get(0); assertEquals(toKill[0].getXid(), info.getXid()); - }, "--tx", "kill", - "xid", toKill[0].getXid().toString(), // Use saved on first run value. - "nodes", grid(0).localNode().consistentId().toString()); + }, "--tx", "--kill", + "--xid", toKill[0].getXid().toString(), // Use saved on first run value. + "--nodes", grid(0).localNode().consistentId().toString()); unlockLatch.countDown(); @@ -662,7 +658,7 @@ public void testKillHangingLocalTransactions() throws Exception { assertEquals(tx0.xid(), info.getXid()); assertEquals(1, map.size()); - }, "--tx", "kill"); + }, "--tx", "--kill"); tx0.finishFuture().get(); @@ -827,7 +823,7 @@ public void testKillHangingRemoteTransactions() throws Exception { // Check kill. validate(h, map -> { // No-op. - }, "--tx", "kill"); + }, "--tx", "--kill"); // Wait for all remote txs to finish. for (Ignite ignite : G.allGrids()) { @@ -885,25 +881,33 @@ public void testBaselineAddOnNotActiveCluster() throws Exception { * @throws Exception If failed. */ public void testCacheHelp() throws Exception { - Ignite ignite = startGrids(1); - - ignite.cluster().active(true); - injectTestSystemOut(); assertEquals(EXIT_CODE_OK, execute("--cache", "help")); for (CacheCommand cmd : CacheCommand.values()) { - if (cmd != CacheCommand.HELP) - assertTrue(cmd.text(), testOut.toString().contains(cmd.text())); + if (cmd != HELP) + assertTrue(cmd.text(), testOut.toString().contains(cmd.toString())); } } + /** + * @throws Exception If failed. + */ + public void testHelp() throws Exception { + injectTestSystemOut(); + + assertEquals(EXIT_CODE_OK, execute("--help")); + + for (Command cmd : Command.values()) + assertTrue(cmd.text(), testOut.toString().contains(cmd.toString())); + } + /** * @throws Exception If failed. */ public void testCacheIdleVerify() throws Exception { - Ignite ignite = startGrids(2); + IgniteEx ignite = (IgniteEx)startGrids(2); ignite.cluster().active(true); @@ -917,7 +921,7 @@ public void testCacheIdleVerify() throws Exception { HashSet clearKeys = new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6)); - ((IgniteEx)ignite).context().cache().cache(DEFAULT_CACHE_NAME).clearLocallyAll(clearKeys, true, true, true); + ignite.context().cache().cache(DEFAULT_CACHE_NAME).clearLocallyAll(clearKeys, true, true, true); assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify")); @@ -978,7 +982,7 @@ public void testCacheIdleVerifyDump() throws Exception { assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME)); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME)); + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skip-zeros", DEFAULT_CACHE_NAME)); Matcher fileNameMatcher = dumpFileNameMatcher(); @@ -1045,14 +1049,16 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception { createCacheAndPreload(ignite, 100); - corruptingAndCheckDefaultCache(ignite, CacheFilterEnum.ALL); + injectTestSystemOut(); + + corruptingAndCheckDefaultCache(ignite, "ALL"); } /** * @param ignite Ignite. - * @param cacheFilterEnum Filter enum. + * @param cacheFilter cacheFilter. */ - private void corruptingAndCheckDefaultCache(IgniteEx ignite, CacheFilterEnum cacheFilterEnum) throws IOException { + private void corruptingAndCheckDefaultCache(IgniteEx ignite, String cacheFilter) throws IOException { injectTestSystemOut(); GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); @@ -1061,7 +1067,7 @@ private void corruptingAndCheckDefaultCache(IgniteEx ignite, CacheFilterEnum cac corruptDataEntry(cacheCtx, cacheCtx.config().getAffinity().partitions() / 2, false, true); - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", cacheFilterEnum.toString())); + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cache-filter", cacheFilter)); Matcher fileNameMatcher = dumpFileNameMatcher(); @@ -1074,129 +1080,6 @@ private void corruptingAndCheckDefaultCache(IgniteEx ignite, CacheFilterEnum cac fail("Should be found dump with conflicts"); } - /** - * Tests that idle verify print partitions info over system caches. - * - * @throws Exception If failed. - */ - public void testCacheIdleVerifyDumpForCorruptedDataOnSystemCache() throws Exception { - int parts = 32; - - atomicConfiguration = new AtomicConfiguration() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(2); - - IgniteEx ignite = (IgniteEx)startGrids(3); - - ignite.cluster().active(true); - - injectTestSystemOut(); - - // Adding some assignments without deployments. - for (int i = 0; i < 100; i++) { - ignite.semaphore("s" + i, i, false, true); - - ignite.atomicSequence("sq" + i, 0, true) - .incrementAndGet(); - } - - CacheGroupContext storedSysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-ds-group")); - - assertNotNull(storedSysCacheCtx); - - corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq0", - "default-ds-group"), true, false); - - corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq" + parts / 2, - "default-ds-group"), false, true); - - CacheGroupContext memorySysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-volatile-ds-group")); - - assertNotNull(memorySysCacheCtx); - - corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s0", - "default-volatile-ds-group"), true, false); - - corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s" + parts / 2, - "default-volatile-ds-group"), false, true); - - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum.SYSTEM.toString())); - - Matcher fileNameMatcher = dumpFileNameMatcher(); - - if (fileNameMatcher.find()) { - String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); - - assertTrue(dumpWithConflicts.contains("found 4 conflict partitions: [counterConflicts=2, " + - "hashConflicts=2]")); - } - else - fail("Should be found dump with conflicts"); - } - - /** - * Tests that idle verify print partitions info over persistence client caches. - * - * @throws Exception If failed. - */ - public void testCacheIdleVerifyDumpForCorruptedDataOnPersistenceClientCache() throws Exception { - IgniteEx ignite = (IgniteEx)startGrids(3); - - ignite.cluster().active(true); - - createCacheAndPreload(ignite, 100); - - corruptingAndCheckDefaultCache(ignite, CacheFilterEnum.PERSISTENT); - } - - /** - * Tests that idle verify print partitions info over none-persistence client caches. - * - * @throws Exception If failed. - */ - public void testCacheIdleVerifyDumpForCorruptedDataOnNonePersistenceClientCache() throws Exception { - int parts = 32; - - dataRegionConfiguration = new DataRegionConfiguration() - .setName("none-persistence-region"); - - IgniteEx ignite = (IgniteEx)startGrids(3); - - ignite.cluster().active(true); - - IgniteCache cache = ignite.createCache(new CacheConfiguration<>() - .setAffinity(new RendezvousAffinityFunction(false, parts)) - .setBackups(2) - .setName(DEFAULT_CACHE_NAME) - .setDataRegionName("none-persistence-region")); - - // Adding some assignments without deployments. - for (int i = 0; i < 100; i++) - cache.put(i, i); - - injectTestSystemOut(); - - GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); - - corruptDataEntry(cacheCtx, 0, true, false); - - corruptDataEntry(cacheCtx, parts / 2, false, true); - - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cacheFilter", CacheFilterEnum - .NOT_PERSISTENT.toString())); - - Matcher fileNameMatcher = dumpFileNameMatcher(); - - if (fileNameMatcher.find()) { - String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); - - assertTrue(dumpWithConflicts.contains("found 1 conflict partitions: [counterConflicts=0, " + - "hashConflicts=1]")); - } - else - fail("Should be found dump with conflicts"); - } - /** * Tests that idle verify print partitions info when node failing. * @@ -1263,9 +1146,9 @@ public void testCacheIdleVerifyDumpWhenSeveralNodesFailing() throws Exception { injectTestSystemOut(); - IgniteInternalFuture fut = GridTestUtils.runAsync(() -> { - assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")); - }); + IgniteInternalFuture fut = GridTestUtils.runAsync( + () -> assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump")) + ); List unstableNodeIds = new ArrayList<>(nodes / 2); @@ -1321,6 +1204,131 @@ private void checkExceptionMessageOnReport(UUID unstableNodeId) throws IOExcepti fail("Should be found dump with conflicts"); } + /** + * Tests that idle verify print partitions info over system caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnSystemCache() throws Exception { + int parts = 32; + + atomicConfiguration = new AtomicConfiguration() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + injectTestSystemOut(); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) { + ignite.semaphore("s" + i, i, false, true); + + ignite.atomicSequence("sq" + i, 0, true) + .incrementAndGet(); + } + + CacheGroupContext storedSysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-ds-group")); + + assertNotNull(storedSysCacheCtx); + + corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq0", + "default-ds-group"), true, false); + + corruptDataEntry(storedSysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("sq" + parts / 2, + "default-ds-group"), false, true); + + CacheGroupContext memorySysCacheCtx = ignite.context().cache().cacheGroup(CU.cacheId("default-volatile-ds-group")); + + assertNotNull(memorySysCacheCtx); + + corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s0", + "default-volatile-ds-group"), true, false); + + corruptDataEntry(memorySysCacheCtx.caches().get(0), new GridCacheInternalKeyImpl("s" + parts / 2, + "default-volatile-ds-group"), false, true); + + assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--cache-filter", "SYSTEM")); + + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("found 4 conflict partitions: [counterConflicts=2, " + + "hashConflicts=2]")); + } + else + fail("Should be found dump with conflicts"); + } + + /** + * Tests that idle verify print partitions info over persistence client caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnPersistenceClientCache() throws Exception { + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + createCacheAndPreload(ignite, 100); + + corruptingAndCheckDefaultCache(ignite, "PERSISTENT"); + } + + /** + * Tests that idle verify print partitions info over none-persistence client caches. + * + * @throws Exception If failed. + */ + public void testCacheIdleVerifyDumpForCorruptedDataOnNonePersistenceClientCache() throws Exception { + int parts = 32; + + dataRegionConfiguration = new DataRegionConfiguration() + .setName("none-persistence-region"); + + IgniteEx ignite = (IgniteEx)startGrids(3); + + ignite.cluster().active(true); + + IgniteCache cache = ignite.createCache(new CacheConfiguration<>() + .setAffinity(new RendezvousAffinityFunction(false, parts)) + .setBackups(2) + .setName(DEFAULT_CACHE_NAME) + .setDataRegionName("none-persistence-region")); + + // Adding some assignments without deployments. + for (int i = 0; i < 100; i++) + cache.put(i, i); + + injectTestSystemOut(); + + GridCacheContext cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context(); + + corruptDataEntry(cacheCtx, 0, true, false); + + corruptDataEntry(cacheCtx, parts / 2, false, true); + + assertEquals( + EXIT_CODE_OK, + execute("--cache", "idle_verify", "--dump", "--cache-filter", "NOT_PERSISTENT") + ); + + Matcher fileNameMatcher = dumpFileNameMatcher(); + + if (fileNameMatcher.find()) { + String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1)))); + + assertTrue(dumpWithConflicts.contains("found 1 conflict partitions: [counterConflicts=0, " + + "hashConflicts=1]")); + } + else + fail("Should be found dump with conflicts"); + } + /** * @return Build matcher for dump file name. */ @@ -1548,16 +1556,16 @@ public void testCacheConfigMultiLineOutputFormatTwoNodeManyCaches() throws Excep /** */ private void testCacheConfigSingleLineOutputFormat(int nodesCnt, int cachesCnt) throws Exception { - testCacheConfig(SINGLE_LINE, nodesCnt, cachesCnt); + testCacheConfig("single-line", nodesCnt, cachesCnt); } /** */ private void testCacheConfigMultiLineOutputFormat(int nodesCnt, int cachesCnt) throws Exception { - testCacheConfig(MULTI_LINE, nodesCnt, cachesCnt); + testCacheConfig("multi-line", nodesCnt, cachesCnt); } /** */ - private void testCacheConfig(OutputFormat outputFormat, int nodesCnt, int cachesCnt) throws Exception { + private void testCacheConfig(String outputFormat, int nodesCnt, int cachesCnt) throws Exception { assertTrue("Invalid number of nodes or caches", nodesCnt > 0 && cachesCnt > 0); Ignite ignite = startGrid(nodesCnt); @@ -1589,26 +1597,28 @@ private void testCacheConfig(OutputFormat outputFormat, int nodesCnt, int caches if (outputFormat == null) exitCode = execute("--cache", "list", ".*", "--config"); else - exitCode = execute("--cache", "list", ".*", "--config", "--output-format", outputFormat.text()); + exitCode = execute("--cache", "list", ".*", "--config", "--output-format", outputFormat); assertEquals(EXIT_CODE_OK, exitCode); String outStr = testOut.toString(); - if (outputFormat == null || outputFormat == SINGLE_LINE) { + if (outputFormat == null || SINGLE_LINE.text().equals(outputFormat)) { for (int i = 0; i < cachesCnt; i++) assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME + i)); assertTrue(outStr.contains("partitions=32")); assertTrue(outStr.contains("function=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction")); } - else if (outputFormat == MULTI_LINE) { + else if (MULTI_LINE.text().equals(outputFormat)) { for (int i = 0; i < cachesCnt; i++) assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + i + "']")); assertTrue(outStr.contains("Affinity Partitions: 32")); assertTrue(outStr.contains("Affinity Function: o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction")); } + else + fail("Unknown output format: " + outputFormat); } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java index dc5011ffe170b..9628d968e45a2 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java @@ -88,8 +88,8 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception { "--cache", "validate_indexes", CACHE_NAME, - "checkFirst", "10000", - "checkThrough", "10")); + "--check-first", "10000", + "--check-through", "10")); assertTrue(testOut.toString().contains("issues found (listed above)")); @@ -159,7 +159,7 @@ private Ignite prepareGridForTest() throws Exception{ ThreadLocalRandom rand = ThreadLocalRandom.current(); - try (IgniteDataStreamer streamer = client.dataStreamer(CACHE_NAME);) { + try (IgniteDataStreamer streamer = client.dataStreamer(CACHE_NAME)) { for (int i = 0; i < 10_000; i++) streamer.addData(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong()))); } From baf9bb4a3bf05ac1af7552825ab35db979722b42 Mon Sep 17 00:00:00 2001 From: Sergey Antonov Date: Mon, 24 Dec 2018 14:33:06 +0300 Subject: [PATCH 267/403] IGNITE-10130 Added an option to disable cache interceptor trigger in case of conflicts - Fixes #5251. Signed-off-by: Alexey Goncharuk (cherry picked from commit 7414c9956d9f00e6514a508035156f2605df286a) --- .../apache/ignite/IgniteSystemProperties.java | 7 + .../processors/cache/GridCacheContext.java | 12 ++ .../processors/cache/GridCacheMapEntry.java | 168 +++++++++++------- 3 files changed, 120 insertions(+), 67 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 9384ee749ad10..3a65f5d42f5b7 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1048,6 +1048,13 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_RECOVERY_VERBOSE_LOGGING = "IGNITE_RECOVERY_VERBOSE_LOGGING"; + /** + * Disables cache interceptor triggering in case of conflicts. + * + * Default is {@code false}. + */ + public static final String IGNITE_DISABLE_TRIGGERING_CACHE_INTERCEPTOR_ON_CONFLICT = "IGNITE_DISABLE_TRIGGERING_CACHE_INTERCEPTOR_ON_CONFLICT"; + /** * When set to {@code true}, cache metrics are not included into the discovery metrics update message (in this * case message contains only cluster metrics). By default cache metrics are included into the message and diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 542f8125a18f5..ef33e8b4928a5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -110,6 +110,7 @@ import org.apache.ignite.plugin.security.SecurityPermission; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISABLE_TRIGGERING_CACHE_INTERCEPTOR_ON_CONFLICT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_READ_LOAD_BALANCING; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -277,6 +278,10 @@ public class GridCacheContext implements Externalizable { /** Recovery mode flag. */ private volatile boolean recoveryMode; + /** */ + private final boolean disableTriggeringCacheInterceptorOnConflict = + Boolean.parseBoolean(System.getProperty(IGNITE_DISABLE_TRIGGERING_CACHE_INTERCEPTOR_ON_CONFLICT, "false")); + /** * Empty constructor required for {@link Externalizable}. */ @@ -865,6 +870,13 @@ public boolean transactionalSnapshot() { return cacheCfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT; } + /** + * @return {@code True} if cache interceptor should be skipped in case of conflicts. + */ + public boolean disableTriggeringCacheInterceptorOnConflict() { + return disableTriggeringCacheInterceptorOnConflict; + } + /** * @return Local node. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index a99cd90604bce..f276297df8027 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -34,6 +34,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cache.CacheInterceptor; import org.apache.ignite.cache.eviction.EvictableEntry; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.NodeStoppingException; @@ -1158,17 +1159,17 @@ else if (res.resultType() == ResultType.FILTERED) { assert !invoke || res.invokeResult() != null; - if(invoke) // No-op invoke happened. + if (invoke) // No-op invoke happened. updRes.invokeResult(res.invokeResult()); updRes.filtered(true); - if(retVal) + if (retVal) updRes.prevValue(res.oldValue()); return updRes; } - else if(noCreate && !invoke && res.resultType() == ResultType.PREV_NULL) + else if (noCreate && !invoke && res.resultType() == ResultType.PREV_NULL) return new GridCacheUpdateTxResult(false); else if (res.resultType() == ResultType.LOCKED) { unlockEntry(); @@ -1263,7 +1264,7 @@ else if (res.resultType() == ResultType.REMOVED_NOT_NULL) { updRes.newValue(res.newValue()); - if(invoke) { + if (invoke) { assert res.invokeResult() != null; updRes.invokeResult(res.invokeResult()); @@ -1370,7 +1371,7 @@ else if (res.resultType() == ResultType.LOCKED) { GridCacheUpdateTxResult updRes = valid ? new GridCacheUpdateTxResult(true, 0L, logPtr) : new GridCacheUpdateTxResult(false, logPtr); - if(retVal && (res.resultType() == ResultType.PREV_NOT_NULL || res.resultType() == ResultType.VERSION_FOUND)) + if (retVal && (res.resultType() == ResultType.PREV_NOT_NULL || res.resultType() == ResultType.VERSION_FOUND)) updRes.prevValue(res.oldValue()); if (needOldVal && compareIgnoreOpCounter(res.resultVersion(), mvccVer) != 0 && @@ -1521,17 +1522,18 @@ else if (res.resultType() == ResultType.LOCKED) { old = oldValPresent ? oldVal : this.val; + if (intercept) + intercept = !skipInterceptor(explicitVer); + if (intercept) { val0 = cctx.unwrapBinaryIfNeeded(val, keepBinary, false); CacheLazyEntry e = new CacheLazyEntry(cctx, key, old, keepBinary); - Object interceptorVal = cctx.config().getInterceptor().onBeforePut( - new CacheLazyEntry(cctx, key, old, keepBinary), - val0); - key0 = e.key(); + Object interceptorVal = cctx.config().getInterceptor().onBeforePut(e, val0); + if (interceptorVal == null) return new GridCacheUpdateTxResult(false, logPtr); else if (interceptorVal != val0) @@ -1749,6 +1751,9 @@ protected Object keyValue(boolean cpy) { old = oldValPresent ? oldVal : val; + if (intercept) + intercept = !skipInterceptor(explicitVer); + if (intercept) { entry0 = new CacheLazyEntry(cctx, key, old, keepBinary); @@ -2070,12 +2075,12 @@ else if (ttl == CU.TTL_NOT_CHANGED) else updated = (CacheObject)writeObj; - op = updated == null ? GridCacheOperation.DELETE : GridCacheOperation.UPDATE; + op = updated == null ? DELETE : UPDATE; if (intercept) { CacheLazyEntry e; - if (op == GridCacheOperation.UPDATE) { + if (op == UPDATE) { updated0 = value(updated0, updated, keepBinary, false); e = new CacheLazyEntry(cctx, key, key0, old, old0, keepBinary); @@ -2108,7 +2113,7 @@ else if (ttl == CU.TTL_NOT_CHANGED) long ttl = CU.TTL_ETERNAL; long expireTime = CU.EXPIRE_TIME_ETERNAL; - if (op == GridCacheOperation.UPDATE) { + if (op == UPDATE) { if (expiryPlc != null) { ttl = CU.toTtl(hadVal ? expiryPlc.getExpiryForUpdate() : expiryPlc.getExpiryForCreation()); @@ -2126,14 +2131,14 @@ else if (ttl != CU.TTL_ZERO) } if (ttl == CU.TTL_ZERO) { - op = GridCacheOperation.DELETE; + op = DELETE; //If time expired no transformation needed. transformOp = false; } // Try write-through. - if (op == GridCacheOperation.UPDATE) { + if (op == UPDATE) { // Detach value before index update. updated = cctx.kernalContext().cacheObjects().prepareForCache(updated, cctx); @@ -2227,7 +2232,7 @@ else if (op == DELETE && transformOp) } if (intercept) { - if (op == GridCacheOperation.UPDATE) + if (op == UPDATE) cctx.config().getInterceptor().onAfterPut(new CacheLazyEntry(cctx, key, key0, updated, updated0, keepBinary, 0L)); else cctx.config().getInterceptor().onAfterRemove(new CacheLazyEntry(cctx, key, key0, old, old0, keepBinary, 0L)); @@ -2320,7 +2325,9 @@ else if (op == DELETE && transformOp) conflictVer, conflictResolve, intercept, - updateCntr); + updateCntr, + cctx.disableTriggeringCacheInterceptorOnConflict() + ); key.valueBytes(cctx.cacheObjectContext()); @@ -2440,7 +2447,7 @@ else if (updateMetrics && REMOVE_NO_VAL.equals(updateRes.outcome()) keepBinary); } - if (c.op == GridCacheOperation.UPDATE) { + if (c.op == UPDATE) { updateVal = val; assert updateVal != null : c; @@ -2470,7 +2477,7 @@ else if (updateMetrics && REMOVE_NO_VAL.equals(updateRes.outcome()) } } else { - assert c.op == GridCacheOperation.DELETE : c.op; + assert c.op == DELETE : c.op; clearReaders(); @@ -2518,29 +2525,23 @@ else if (updateMetrics && REMOVE_NO_VAL.equals(updateRes.outcome()) topVer); } - if (intercept) { - if (c.op == GridCacheOperation.UPDATE) { - cctx.config().getInterceptor().onAfterPut(new CacheLazyEntry( - cctx, - key, - null, - updateVal, - null, - keepBinary, - c.updateRes.updateCounter())); - } - else { - assert c.op == GridCacheOperation.DELETE : c.op; + if (intercept && c.wasIntercepted) { + assert c.op == UPDATE || c.op == DELETE : c.op; - cctx.config().getInterceptor().onAfterRemove(new CacheLazyEntry( - cctx, - key, - null, - oldVal, - null, - keepBinary, - c.updateRes.updateCounter())); - } + Cache.Entry entry = new CacheLazyEntry<>( + cctx, + key, + null, + c.op == UPDATE ? updateVal : oldVal, + null, + keepBinary, + c.updateRes.updateCounter() + ); + + if (c.op == UPDATE) + cctx.config().getInterceptor().onAfterPut(entry); + else + cctx.config().getInterceptor().onAfterRemove(entry); } } finally { @@ -3283,6 +3284,34 @@ protected final boolean hasValueUnlocked() { return val != null; } + /** + * Checks, that changes were got by DR. + * + * @param explicitVer – Explicit version (if any). + * @return {@code true} if changes were got by DR and {@code false} otherwise. + */ + private boolean isRemoteDrUpdate(@Nullable GridCacheVersion explicitVer) { + return explicitVer != null && explicitVer.dataCenterId() != cctx.dr().dataCenterId(); + } + + /** + * Checks, that cache interceptor should be skipped. + *

+ * It is expects by default behavior that Interceptor methods ({@link CacheInterceptor#onBeforePut(Cache.Entry, + * Object)}, {@link CacheInterceptor#onAfterPut(Cache.Entry)}, {@link CacheInterceptor#onBeforeRemove(Cache.Entry)} + * and {@link CacheInterceptor#onAfterRemove(Cache.Entry)}) will be called, but {@link + * CacheInterceptor#onGet(Object, Object)}. This can even make DR-update flow broken in case of non-idempotent + * Interceptor and force users to call onGet manually as the only workaround. Also, user may want to skip + * Interceptor to avoid redundant entry transformation for DR updates and exchange with internal data b/w data + * centres which is a normal case. + * + * @param explicitVer - Explicit version (if any). + * @return {@code true} if cache interceptor should be skipped and {@code false} otherwise. + */ + private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { + return isRemoteDrUpdate(explicitVer) && cctx.disableTriggeringCacheInterceptorOnConflict(); + } + /** {@inheritDoc} */ @Override public CacheObject rawPut(CacheObject val, long ttl) { lockEntry(); @@ -3466,7 +3495,7 @@ else if (deletedUnlocked()) cctx.cacheId(), key, val, - val == null ? GridCacheOperation.DELETE : GridCacheOperation.CREATE, + val == null ? DELETE : GridCacheOperation.CREATE, null, ver, expireTime, @@ -3479,7 +3508,7 @@ else if (deletedUnlocked()) cctx.cacheId(), key, val, - val == null ? GridCacheOperation.DELETE : GridCacheOperation.CREATE, + val == null ? DELETE : GridCacheOperation.CREATE, null, ver, expireTime, @@ -4345,9 +4374,9 @@ protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expi if (tx.local()) { // For remote tx we log all updates in batch: GridDistributedTxRemoteAdapter.commitIfLocked() GridCacheOperation op; if (val == null) - op = GridCacheOperation.DELETE; + op = DELETE; else - op = this.val == null ? GridCacheOperation.CREATE : GridCacheOperation.UPDATE; + op = this.val == null ? GridCacheOperation.CREATE : UPDATE; return cctx.shared().wal().log(new DataRecord(new DataEntry( cctx.cacheId(), @@ -4381,9 +4410,9 @@ protected WALPointer logMvccUpdate(IgniteInternalTx tx, CacheObject val, long ex if (tx.local()) { // For remote tx we log all updates in batch: GridDistributedTxRemoteAdapter.commitIfLocked() GridCacheOperation op; if (val == null) - op = GridCacheOperation.DELETE; + op = DELETE; else - op = this.val == null ? GridCacheOperation.CREATE : GridCacheOperation.UPDATE; + op = this.val == null ? GridCacheOperation.CREATE : UPDATE; return cctx.shared().wal().log(new MvccDataRecord(new MvccDataEntry( cctx.cacheId(), @@ -4958,7 +4987,7 @@ protected final void checkOwnerChanged(@Nullable CacheLockCandidates prevOwners, */ private void updateMetrics(GridCacheOperation op, boolean metrics, boolean transformed, boolean hasOldVal) { if (metrics && cctx.statisticsEnabled()) { - if (op == GridCacheOperation.DELETE) { + if (op == DELETE) { cctx.cache().metrics0().onRemove(); if (transformed) @@ -5535,7 +5564,7 @@ else if (res.resultType() == ResultType.FILTERED) { updRes.filtered(true); - if(needVal) + if (needVal) updRes.prevValue(res.oldValue()); resFut.onDone(updRes); @@ -5619,7 +5648,7 @@ else if (res.resultType() == ResultType.REMOVED_NOT_NULL) { GridCacheUpdateTxResult updRes = valid ? new GridCacheUpdateTxResult(true, 0L, logPtr) : new GridCacheUpdateTxResult(false, logPtr); - if(invoke) { + if (invoke) { assert res.invokeResult() != null; updRes.invokeResult(res.invokeResult()); @@ -5895,6 +5924,9 @@ private static class AtomicCacheUpdateClosure implements IgniteCacheOffheapManag /** */ private final Long updateCntr; + /** */ + private final boolean skipInterceptorOnConflict; + /** */ private GridCacheUpdateAtomicResult updateRes; @@ -5908,8 +5940,12 @@ private static class AtomicCacheUpdateClosure implements IgniteCacheOffheapManag private CacheDataRow oldRow; /** OldRow expiration flag. */ - private boolean oldRowExpiredFlag = false; + private boolean oldRowExpiredFlag; + + /** Disable interceptor invocation onAfter* methods flag. */ + private boolean wasIntercepted; + /** */ AtomicCacheUpdateClosure( GridCacheMapEntry entry, AffinityTopologyVersion topVer, @@ -5929,7 +5965,8 @@ private static class AtomicCacheUpdateClosure implements IgniteCacheOffheapManag @Nullable GridCacheVersion conflictVer, boolean conflictResolve, boolean intercept, - @Nullable Long updateCntr) { + @Nullable Long updateCntr, + boolean skipInterceptorOnConflict) { assert op == UPDATE || op == DELETE || op == TRANSFORM : op; this.entry = entry; @@ -5951,6 +5988,7 @@ private static class AtomicCacheUpdateClosure implements IgniteCacheOffheapManag this.conflictResolve = conflictResolve; this.intercept = intercept; this.updateCntr = updateCntr; + this.skipInterceptorOnConflict = skipInterceptorOnConflict; switch (op) { case UPDATE: @@ -6308,7 +6346,7 @@ private void update(@Nullable GridCacheVersionConflictContext conflictCtx, newExpireTime = entry.expireTimeExtras(); } else if (newSysTtl == CU.TTL_ZERO) { - op = GridCacheOperation.DELETE; + op = DELETE; writeObj = null; @@ -6328,18 +6366,16 @@ else if (newSysTtl == CU.TTL_ZERO) { newSysExpireTime = newExpireTime = conflictCtx.expireTime(); } - if (intercept) { + if (intercept && (conflictVer == null || !skipInterceptorOnConflict)) { Object updated0 = cctx.unwrapBinaryIfNeeded(updated, keepBinary, false); - CacheLazyEntry interceptEntry = new CacheLazyEntry<>(cctx, - entry.key, - null, - oldVal, - null, - keepBinary); + CacheLazyEntry interceptEntry = + new CacheLazyEntry<>(cctx, entry.key, null, oldVal, null, keepBinary); Object interceptorVal = cctx.config().getInterceptor().onBeforePut(interceptEntry, updated0); + wasIntercepted = true; + if (interceptorVal == null) { treeOp = IgniteTree.OperationType.NOOP; @@ -6438,16 +6474,14 @@ private void remove(@Nullable GridCacheVersionConflictContext conflictCtx, IgniteBiTuple interceptRes = null; - if (intercept) { - CacheLazyEntry intercepEntry = new CacheLazyEntry<>(cctx, - entry.key, - null, - oldVal, - null, - keepBinary); + if (intercept && (conflictVer == null || !skipInterceptorOnConflict)) { + CacheLazyEntry intercepEntry = + new CacheLazyEntry<>(cctx, entry.key, null, oldVal, null, keepBinary); interceptRes = cctx.config().getInterceptor().onBeforeRemove(intercepEntry); + wasIntercepted = true; + if (cctx.cancelRemove(interceptRes)) { treeOp = IgniteTree.OperationType.NOOP; @@ -6611,7 +6645,7 @@ private void remove(@Nullable GridCacheVersionConflictContext conflictCtx, assert conflictCtx.isUseNew(); // Update value is known at this point, so update operation type. - op = writeObj != null ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE; + op = writeObj != null ? UPDATE : DELETE; } return conflictCtx; From 533fc2b49d753929e8f26d299c0dcee4f72687d7 Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Mon, 24 Dec 2018 12:49:58 +0300 Subject: [PATCH 268/403] IGNITE-10671 Fixed double initialization of FileArchiver leading to broken WAL file compression - Fixes #5665. Signed-off-by: Alexey Goncharuk (cherry picked from commit e87aa510a74e4a6efb4dd72544a6419799307f9a) --- .../wal/FileWriteAheadLogManager.java | 246 ++++++++++-------- .../wal/aware/SegmentArchivedStorage.java | 7 + .../persistence/wal/aware/SegmentAware.java | 11 + .../wal/aware/SegmentCompressStorage.java | 7 + .../wal/aware/SegmentCurrentStateStorage.java | 9 + .../db/wal/WalCompactionSwitchOnTest.java | 142 ++++++++++ .../testsuites/IgnitePdsTestSuite2.java | 2 + 7 files changed, 317 insertions(+), 107 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionSwitchOnTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 85139a750dfef..4e935177c7a75 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -185,13 +185,13 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private static final FileDescriptor[] EMPTY_DESCRIPTORS = new FileDescriptor[0]; - /** */ + /** Zero-filled buffer for file formatting. */ private static final byte[] FILL_BUF = new byte[1024 * 1024]; - /** Pattern for segment file names */ + /** Pattern for segment file names. */ public static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal"); - /** */ + /** Pattern for WAL temp files - these files will be cleared at startup. */ public static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp"); /** WAL segment file filter, see {@link #WAL_NAME_PATTERN} */ @@ -201,7 +201,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl } }; - /** */ + /** WAL segment temporary file filter, see {@link #WAL_TEMP_NAME_PATTERN} */ private static final FileFilter WAL_SEGMENT_TEMP_FILE_FILTER = new FileFilter() { @Override public boolean accept(File file) { return !file.isDirectory() && WAL_TEMP_NAME_PATTERN.matcher(file.getName()).matches(); @@ -278,7 +278,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl * Maximum number of allowed segments without checkpoint. If we have their more checkpoint should be triggered. * It is simple way to calculate WAL size without checkpoint instead fair WAL size calculating. */ - private long maxSegCountWithoutCheckpoint; + private final long maxSegCountWithoutCheckpoint; /** Size of wal archive since which removing of old archive should be started */ private final long allowedThresholdWalArchiveSize; @@ -348,7 +348,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private final ThreadLocal lastWALPtr = new ThreadLocal<>(); - /** Current log segment handle */ + /** Current log segment handle. */ private volatile FileWriteHandle currHnd; /** */ @@ -365,7 +365,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl * segment, skip possible archiving for this case
Value is filled only for case {@link * #walAutoArchiveAfterInactivity} > 0
*/ - private AtomicLong lastRecordLoggedMs = new AtomicLong(); + private final AtomicLong lastRecordLoggedMs = new AtomicLong(); /** * Cancellable task for {@link WALMode#BACKGROUND}, should be cancelled at shutdown. @@ -380,7 +380,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl @Nullable private volatile GridTimeoutObject nextAutoArchiveTimeoutObj; /** WAL writer worker. */ - private WALWriter walWriter; + private volatile WALWriter walWriter; /** * Listener invoked for each segment file IO initializer. @@ -396,7 +396,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private SegmentRouter segmentRouter; /** Segment factory with ability locked segment during reading. */ - private SegmentFileInputFactory lockedSegmentFileInputFactory; + private volatile SegmentFileInputFactory lockedSegmentFileInputFactory; /** * @param ctx Kernal context. @@ -424,6 +424,10 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { evt = ctx.event(); failureProcessor = ctx.failure(); segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled()); + + maxSegCountWithoutCheckpoint = + (long)((U.adjustedWalHistorySize(dsCfg, log) * CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE) + / dsCfg.getWalSegmentSize()); } /** @@ -437,117 +441,115 @@ public void setFileIOFactory(FileIOFactory ioFactory) { /** {@inheritDoc} */ @Override public void start0() throws IgniteCheckedException { - if (!cctx.kernalContext().clientNode()) { - maxSegCountWithoutCheckpoint = - (long)((U.adjustedWalHistorySize(dsCfg, log) * CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE) - / dsCfg.getWalSegmentSize()); + if(cctx.kernalContext().clientNode()) + return; - final PdsFolderSettings resolveFolders = cctx.kernalContext().pdsFolderResolver().resolveFolders(); + final PdsFolderSettings resolveFolders = cctx.kernalContext().pdsFolderResolver().resolveFolders(); - checkWalConfiguration(); + walWriter = new WALWriter(log); + + checkWalConfiguration(); - final File walWorkDir0 = walWorkDir = initDirectory( + final File walWorkDir0 = walWorkDir = initDirectory( dsCfg.getWalPath(), DataStorageConfiguration.DFLT_WAL_PATH, resolveFolders.folderName(), "write ahead log work directory" - ); + ); - final File walArchiveDir0 = walArchiveDir = initDirectory( + final File walArchiveDir0 = walArchiveDir = initDirectory( dsCfg.getWalArchivePath(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, resolveFolders.folderName(), "write ahead log archive directory" - ); + ); - serializer = new RecordSerializerFactoryImpl(cctx).createSerializer(serializerVer); + serializer = new RecordSerializerFactoryImpl(cctx).createSerializer(serializerVer); - GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.database(); + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.database(); - metrics = dbMgr.persistentStoreMetricsImpl(); + metrics = dbMgr.persistentStoreMetricsImpl(); - checkOrPrepareFiles(); + checkOrPrepareFiles(); - if (metrics != null) - metrics.setWalSizeProvider(new CO() { - @Override public Long apply() { - long size = 0; + if (metrics != null) + metrics.setWalSizeProvider(new CO() { + @Override public Long apply() { + long size = 0; - for (File f : walWorkDir0.listFiles()) - size += f.length(); + for (File f : walWorkDir0.listFiles()) + size += f.length(); - for (File f : walArchiveDir0.listFiles()) - size += f.length(); + for (File f : walArchiveDir0.listFiles()) + size += f.length(); - return size; - } - }); - - IgniteBiTuple tup = scanMinMaxArchiveIndices(); - - segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled()); - - segmentAware.lastTruncatedArchiveIdx(tup == null ? -1 : tup.get1() - 1); + return size; + } + }); - long lastAbsArchivedIdx = tup == null ? -1 : tup.get2(); + IgniteBiTuple tup = scanMinMaxArchiveIndices(); - if (isArchiverEnabled()) - archiver = new FileArchiver(lastAbsArchivedIdx, log); - else - archiver = null; + segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled()); - if (lastAbsArchivedIdx > 0) - segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx); + segmentAware.lastTruncatedArchiveIdx(tup == null ? -1 : tup.get1() - 1); - if (dsCfg.isWalCompactionEnabled()) { - if (compressor == null) - compressor = new FileCompressor(log); + long lastAbsArchivedIdx = tup == null ? -1 : tup.get2(); - if (decompressor == null) { // Preventing of two file-decompressor thread instantiations. - decompressor = new FileDecompressor(log); + if (isArchiverEnabled()) + archiver = new FileArchiver(lastAbsArchivedIdx, log); + else + archiver = null; - new IgniteThread(decompressor).start(); - } - } + if (lastAbsArchivedIdx > 0) + segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx); - segmentRouter = new SegmentRouter(walWorkDir, walArchiveDir, segmentAware, dsCfg); + if (dsCfg.isWalCompactionEnabled()) { + compressor = new FileCompressor(log); - walDisableContext = cctx.walState().walDisableContext(); + decompressor = new FileDecompressor(log); + } - if (mode != WALMode.NONE && mode != WALMode.FSYNC) { - walSegmentSyncWorker = new WalSegmentSyncer(igCfg.getIgniteInstanceName(), - cctx.kernalContext().log(WalSegmentSyncer.class)); + segmentRouter = new SegmentRouter(walWorkDir, walArchiveDir, segmentAware, dsCfg); - if (log.isInfoEnabled()) - log.info("Started write-ahead log manager [mode=" + mode + ']'); - } - else - U.quietAndWarn(log, "Started write-ahead log manager in NONE mode, persisted data may be lost in " + - "a case of unexpected node failure. Make sure to deactivate the cluster before shutdown."); + if (mode != WALMode.NONE && mode != WALMode.FSYNC) { + walSegmentSyncWorker = new WalSegmentSyncer(igCfg.getIgniteInstanceName(), + cctx.kernalContext().log(WalSegmentSyncer.class)); - lockedSegmentFileInputFactory = new LockedSegmentFileInputFactory( - segmentAware, - segmentRouter, - ioFactory - ); + if (log.isInfoEnabled()) + log.info("Started write-ahead log manager [mode=" + mode + ']'); } + else + U.quietAndWarn(log, "Started write-ahead log manager in NONE mode, persisted data may be lost in " + + "a case of unexpected node failure. Make sure to deactivate the cluster before shutdown."); + + lockedSegmentFileInputFactory = new LockedSegmentFileInputFactory( + segmentAware, + segmentRouter, + ioFactory + ); } /** * */ private void startArchiverAndCompressor() { + segmentAware.reset(); + if (isArchiverEnabled()) { - assert archiver != null; + assert archiver != null : "FileArchiver should be initialized."; - new IgniteThread(archiver).start(); + archiver.restart(); } - if (walSegmentSyncWorker != null) - new IgniteThread(walSegmentSyncWorker).start(); + if (dsCfg.isWalCompactionEnabled()) { + assert compressor != null : "Compressor should be initialized."; - if (compressor != null) - compressor.start(); + compressor.restart(); + + assert decompressor != null : "Compressor should be initialized."; + + decompressor.restart(); + } } /** @@ -615,7 +617,10 @@ private void checkWalConfiguration() throws IgniteCheckedException { } } - /** {@inheritDoc} */ + /** + * Method is called twice on deactivate and stop. + * It shutdown workers but do not deallocate them to avoid duplication. + * */ @Override protected void stop0(boolean cancel) { final GridTimeoutProcessor.CancelableTask schedule = backgroundFlushSchedule; @@ -649,17 +654,11 @@ private void checkWalConfiguration() throws IgniteCheckedException { if (archiver != null) archiver.shutdown(); - if (compressor != null) { + if (compressor != null) compressor.shutdown(); - compressor = null; - } - - if (decompressor != null) { + if (decompressor != null) decompressor.shutdown(); - - decompressor = null; - } } catch (Exception e) { U.error(log, "Failed to gracefully close WAL segment: " + this.currHnd.fileIO, e); @@ -671,8 +670,7 @@ private void checkWalConfiguration() throws IgniteCheckedException { if (log.isDebugEnabled()) log.debug("Activated file write ahead log manager [nodeId=" + cctx.localNodeId() + " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); - - start0(); + //NOOP implementation, we need to override it. } /** {@inheritDoc} */ @@ -698,6 +696,15 @@ private void checkWalConfiguration() throws IgniteCheckedException { /** {@inheritDoc} */ @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException { + if (log.isDebugEnabled()) + log.debug("File write ahead log manager resuming logging [nodeId=" + cctx.localNodeId() + + " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); + + /* + walDisableContext is started after FileWriteAheadLogManager, so we obtain actual walDisableContext ref here. + */ + walDisableContext = cctx.walState().walDisableContext(); + assert currHnd == null; assert lastPtr == null || lastPtr instanceof FileWALPointer; @@ -708,10 +715,8 @@ private void checkWalConfiguration() throws IgniteCheckedException { FileWALPointer filePtr = (FileWALPointer)lastPtr; - walWriter = new WALWriter(log); - if (!mmap) - new IgniteThread(walWriter).start(); + walWriter.restart(); currHnd = restoreWriteHandle(filePtr); @@ -730,13 +735,8 @@ private void checkWalConfiguration() throws IgniteCheckedException { currHnd.resume = false; - if (mode == WALMode.BACKGROUND) { - backgroundFlushSchedule = cctx.time().schedule(new Runnable() { - @Override public void run() { - doFlush(); - } - }, flushFreq, flushFreq); - } + if (mode == WALMode.BACKGROUND) + backgroundFlushSchedule = cctx.time().schedule(this::doFlush, flushFreq, flushFreq); if (walAutoArchiveAfterInactivity > 0) scheduleNextInactivityPeriodElapsedCheck(); @@ -1977,6 +1977,17 @@ private void allocateRemainingFiles() throws StorageException { } ); } + + /** + * Restart worker in IgniteThread. + */ + public void restart() { + assert runner() == null : "FileArchiver is still running"; + + isCancelled = false; + + new IgniteThread(archiver).start(); + } } /** @@ -1985,7 +1996,7 @@ private void allocateRemainingFiles() throws StorageException { */ private class FileCompressor extends FileCompressorWorker { /** Workers queue. */ - List workers = new ArrayList<>(); + private final List workers = new ArrayList<>(); /** */ FileCompressor(IgniteLogger log) { @@ -2011,9 +2022,11 @@ private void init() { for (int i = 1; i < calculateThreadCount(); i++) { FileCompressorWorker worker = new FileCompressorWorker(i, log); - worker.start(); + worker.restart(); - workers.add(worker); + synchronized (this) { + workers.add(worker); + } } } @@ -2064,19 +2077,18 @@ private void shutdown() throws IgniteInterruptedCheckedException { /** */ private class FileCompressorWorker extends GridWorker { - /** */ - private Thread thread; - /** */ FileCompressorWorker(int idx, IgniteLogger log) { super(cctx.igniteInstanceName(), "wal-file-compressor-%" + cctx.igniteInstanceName() + "%-" + idx, log); } /** */ - void start() { - thread = new IgniteThread(this); + void restart() { + assert runner() == null : "FileCompressorWorker is still running."; + + isCancelled = false; - thread.start(); + new IgniteThread(this).start(); } /** @@ -2396,6 +2408,15 @@ private void shutdown() { U.join(this, log); } + + /** Restart worker. */ + void restart() { + assert runner() == null : "FileDecompressor is still running."; + + isCancelled = false; + + new IgniteThread(this).start(); + } } /** @@ -3454,6 +3475,17 @@ else if (err != null) } } + /** + * Restart worker in IgniteThread. + */ + public void restart() { + assert runner() == null : "WALWriter is still running"; + + isCancelled = false; + + new IgniteThread(archiver).start(); + } + /** * Shutdowns thread. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java index e31628f40abbf..57ac848fc8295 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java @@ -124,6 +124,13 @@ synchronized void interrupt() { notifyAll(); } + /** + * Resets interrupted flag. + */ + void reset() { + interrupted = false; + } + /** * Check for interrupt flag was set. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java index a14f0cac3a325..b564aec868bb0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java @@ -238,6 +238,17 @@ public void releaseWorkSegment(long absIdx) { segmentLockStorage.releaseWorkSegment(absIdx); } + /** + * Reset interrupted flag. + */ + public void reset() { + segmentArchivedStorage.reset(); + + segmentCompressStorage.reset(); + + segmentCurrStateStorage.reset(); + } + /** * Interrupt waiting on related objects. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java index 95d4f4a770c39..d93bb8423acc4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java @@ -170,4 +170,11 @@ void keepUncompressedIdxFrom(long idx) { long keepUncompressedIdxFrom() { return minUncompressedIdxToKeep; } + + /** + * Reset interrupted flag. + */ + public void reset() { + interrupted = false; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java index 5761ef9fcbb46..d08a9b8bdeaae 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java @@ -168,4 +168,13 @@ private void checkInterrupted() throws IgniteInterruptedCheckedException { if (interrupted) throw new IgniteInterruptedCheckedException("Interrupt waiting of change current idx"); } + + /** + * Reset interrupted flag. + */ + public void reset() { + interrupted = false; + + forceInterrupted = false; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionSwitchOnTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionSwitchOnTest.java new file mode 100644 index 0000000000000..64a937cbe3a9c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionSwitchOnTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal; + +import java.io.File; +import java.io.FileFilter; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Load without compaction -> Stop -> Enable WAL Compaction -> Start. + */ +public class WalCompactionSwitchOnTest extends GridCommonAbstractTest { + /** Compaction enabled. */ + private boolean compactionEnabled; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(256 * 1024 * 1024)) + .setWalSegmentSize(512 * 1024) + .setWalSegments(100) + .setWalCompactionEnabled(compactionEnabled)); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + cleanPersistenceDir(); + } + + /** + * Load without compaction -> Stop -> Enable WAL Compaction -> Start. + * + * @throws Exception On exception. + */ + public void testWalCompactionSwitch() throws Exception { + IgniteEx ex = startGrid(0); + + ex.cluster().active(true); + + IgniteCache cache = ex.getOrCreateCache( + new CacheConfiguration() + .setName("c1") + .setGroupName("g1") + .setCacheMode(CacheMode.PARTITIONED) + ); + + for (int i = 0; i < 500; i++) + cache.put(i, i); + + File walDir = U.resolveWorkDirectory( + ex.configuration().getWorkDirectory(), + "db/wal/node00-" + ex.localNode().consistentId(), + false + ); + + forceCheckpoint(); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + File[] archivedFiles = walDir.listFiles(new FileFilter() { + @Override public boolean accept(File pathname) { + return pathname.getName().endsWith(".wal"); + } + }); + + return archivedFiles.length == 39; + } + }, 5000); + + stopGrid(0); + + compactionEnabled = true; + + ex = startGrid(0); + + ex.cluster().active(true); + + File archiveDir = U.resolveWorkDirectory( + ex.configuration().getWorkDirectory(), + "db/wal/archive/node00-" + ex.localNode().consistentId(), + false + ); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + File[] archivedFiles = archiveDir.listFiles(new FileFilter() { + @Override public boolean accept(File pathname) { + return pathname.getName().endsWith(FilePageStoreManager.ZIP_SUFFIX); + } + }); + + return archivedFiles.length == 20; + } + }, 5000); + + File[] tmpFiles = archiveDir.listFiles(new FileFilter() { + @Override public boolean accept(File pathname) { + return pathname.getName().endsWith(FilePageStoreManager.TMP_SUFFIX); + } + }); + + assertEquals(0, tmpFiles.length); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index baaf88254f4f2..6e9391ef623a9 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorExceptionDuringReadTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionSwitchOnTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalDeletionArchiveFsyncTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalDeletionArchiveLogOnlyTest; @@ -173,6 +174,7 @@ public static void addRealPageStoreTests(TestSuite suite) { suite.addTestSuite(IgniteWalSerializerVersionTest.class); suite.addTestSuite(WalCompactionTest.class); + suite.addTestSuite(WalCompactionSwitchOnTest.class); suite.addTestSuite(WalDeletionArchiveFsyncTest.class); suite.addTestSuite(WalDeletionArchiveLogOnlyTest.class); From 9ee6874502e6cb164f49fc6e0f1b53ad70091505 Mon Sep 17 00:00:00 2001 From: Andrey Kuznetsov Date: Mon, 24 Dec 2018 17:38:37 +0300 Subject: [PATCH 269/403] IGNITE-10003 Introduced SYSTEM_CRITICAL_OPERATION_TIMEOUT failure type Signed-off-by: Andrey Gura --- .../failure/AbstractFailureHandler.java | 5 +- .../apache/ignite/failure/FailureType.java | 5 +- .../GridCacheDatabaseSharedManager.java | 5 +- .../failure/SystemWorkersBlockingTest.java | 17 ++- .../CheckpointReadLockFailureTest.java | 125 ++++++++++++++++++ .../IgniteBasicWithPersistenceTestSuite.java | 7 +- 6 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java index d3685c002a1fc..79b1f8f6ea591 100644 --- a/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/failure/AbstractFailureHandler.java @@ -18,11 +18,13 @@ package org.apache.ignite.failure; import java.util.Collections; +import java.util.EnumSet; import java.util.Set; import org.apache.ignite.Ignite; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; +import static org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED; /** @@ -33,7 +35,8 @@ public abstract class AbstractFailureHandler implements FailureHandler { /** */ @GridToStringInclude - private Set ignoredFailureTypes = Collections.singleton(SYSTEM_WORKER_BLOCKED); + private Set ignoredFailureTypes = + Collections.unmodifiableSet(EnumSet.of(SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT)); /** * Sets failure types that must be ignored by failure handler. diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java index fbd5529fc8792..114e432e1ef73 100644 --- a/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java +++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java @@ -31,5 +31,8 @@ public enum FailureType { SYSTEM_WORKER_BLOCKED, /** Critical error - error which leads to the system's inoperability. */ - CRITICAL_ERROR + CRITICAL_ERROR, + + /** System-critical operation has been timed out. */ + SYSTEM_CRITICAL_OPERATION_TIMEOUT } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 0221cb38dec7d..d6c96b0f57d65 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -173,6 +173,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_READ_LOCK_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD; import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; +import static org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD; import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD; @@ -263,7 +264,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan private volatile GridFutureAdapter enableChangeApplied; /** */ - private ReentrantReadWriteLock checkpointLock = new ReentrantReadWriteLock(); + ReentrantReadWriteLock checkpointLock = new ReentrantReadWriteLock(); /** */ private long checkpointFreq; @@ -1555,7 +1556,7 @@ private void failCheckpointReadLock() throws CheckpointReadLockTimeoutException, IgniteException e = new IgniteException(msg); - if (cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e))) + if (cctx.kernalContext().failure().process(new FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, e))) throw e; throw new CheckpointReadLockTimeoutException(msg); diff --git a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java index 3ca7948a38c76..28c5897a2caff 100644 --- a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.failure; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.ignite.Ignite; @@ -40,13 +42,22 @@ public class SystemWorkersBlockingTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setFailureHandler(new AbstractFailureHandler() { + AbstractFailureHandler failureHnd = new AbstractFailureHandler() { @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) { - hndLatch.countDown(); + if (failureCtx.type() == FailureType.SYSTEM_WORKER_BLOCKED) + hndLatch.countDown(); return false; } - }); + }; + + Set ignoredFailureTypes = new HashSet<>(failureHnd.getIgnoredFailureTypes()); + + ignoredFailureTypes.remove(FailureType.SYSTEM_WORKER_BLOCKED); + + failureHnd.setIgnoredFailureTypes(ignoredFailureTypes); + + cfg.setFailureHandler(failureHnd); cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java new file mode 100644 index 0000000000000..2ec5c2d50aee5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointReadLockFailureTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.Ignite; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.AbstractFailureHandler; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests critical failure handling on checkpoint read lock acquisition errors. + */ +public class CheckpointReadLockFailureTest extends GridCommonAbstractTest { + /** */ + private static final AbstractFailureHandler FAILURE_HND = new AbstractFailureHandler() { + @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) { + if (failureCtx.type() != FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT) + return true; + + if (hndLatch != null) + hndLatch.countDown(); + + return false; + } + }; + + /** */ + private static volatile CountDownLatch hndLatch; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setFailureHandler(FAILURE_HND) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true)) + .setCheckpointFrequency(Integer.MAX_VALUE) + .setCheckpointReadLockTimeout(1)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + Set ignoredFailureTypes = new HashSet<>(FAILURE_HND.getIgnoredFailureTypes()); + ignoredFailureTypes.remove(FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT); + + FAILURE_HND.setIgnoredFailureTypes(ignoredFailureTypes); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + cleanPersistenceDir(); + } + + /** + * @throws Exception If failed. + */ + public void testFailureTypeOnTimeout() throws Exception { + hndLatch = new CountDownLatch(1); + + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig.context().cache().context().database(); + + IgniteInternalFuture acquireWriteLock = GridTestUtils.runAsync(() -> { + db.checkpointLock.writeLock().lock(); + + try { + doSleep(Long.MAX_VALUE); + } + finally { + db.checkpointLock.writeLock().unlock(); + } + }); + + GridTestUtils.waitForCondition(() -> db.checkpointLock.writeLock().isHeldByCurrentThread(), 5000); + + IgniteInternalFuture acquireReadLock = GridTestUtils.runAsync(() -> { + db.checkpointReadLock(); + db.checkpointReadUnlock(); + }); + + assertTrue(hndLatch.await(5, TimeUnit.SECONDS)); + + acquireWriteLock.cancel(); + + acquireReadLock.get(5, TimeUnit.SECONDS); + + GridTestUtils.waitForCondition(acquireWriteLock::isCancelled, 5000); + + stopGrid(0); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java index 36f77ccc96912..87bd86f10cc8b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java @@ -18,11 +18,14 @@ package org.apache.ignite.testsuites; import java.util.Set; + import junit.framework.TestSuite; import org.apache.ignite.failure.FailureHandlingConfigurationTest; import org.apache.ignite.failure.IoomFailureHandlerTest; +import org.apache.ignite.failure.SystemWorkersBlockingTest; import org.apache.ignite.failure.SystemWorkersTerminationTest; import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest; +import org.apache.ignite.internal.GridNodeMetricsLogPdsSelfTest; import org.apache.ignite.internal.encryption.EncryptedCacheBigEntryTest; import org.apache.ignite.internal.encryption.EncryptedCacheCreateTest; import org.apache.ignite.internal.encryption.EncryptedCacheDestroyTest; @@ -30,7 +33,7 @@ import org.apache.ignite.internal.encryption.EncryptedCacheNodeJoinTest; import org.apache.ignite.internal.encryption.EncryptedCachePreconfiguredRestartTest; import org.apache.ignite.internal.encryption.EncryptedCacheRestartTest; -import org.apache.ignite.internal.GridNodeMetricsLogPdsSelfTest; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointReadLockFailureTest; import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest; import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest; import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest; @@ -66,6 +69,8 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex suite.addTestSuite(GridMarshallerMappingConsistencyTest.class); suite.addTestSuite(SystemWorkersTerminationTest.class); suite.addTestSuite(FailureHandlingConfigurationTest.class); + suite.addTestSuite(SystemWorkersBlockingTest.class); + suite.addTestSuite(CheckpointReadLockFailureTest.class); suite.addTestSuite(GridCommandHandlerTest.class); suite.addTestSuite(GridCommandHandlerSslTest.class); From e43765fefd79063577b8ce98990dcb56257fb748 Mon Sep 17 00:00:00 2001 From: Pavel Voronkin Date: Mon, 24 Dec 2018 12:49:58 +0300 Subject: [PATCH 270/403] IGNITE-10671 Fixed tests after merge. Signed-off-by: Alexey Goncharuk (cherry picked from commit e87aa510a74e4a6efb4dd72544a6419799307f9a) --- .../persistence/wal/FileWriteAheadLogManager.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 4e935177c7a75..03a669b2fd775 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -3492,9 +3492,15 @@ public void restart() { public void shutdown() throws IgniteInterruptedCheckedException { U.cancel(this); - LockSupport.unpark(runner()); + Thread runner = runner(); - U.join(runner()); + if (runner != null) { + LockSupport.unpark(runner); + + U.join(runner); + } + + assert runner() == null : "WALWriter should be stopped."; } /** From d58b88679eb25cf7ebf91ed5dfe99a7b700f8e76 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Tue, 25 Dec 2018 18:08:17 +0300 Subject: [PATCH 271/403] IGNITE-10558: MVCC: IgniteWalReader fixed. This closes #5583. (cherry picked from commit 417fae51f79c7fbc9f17943f0df1ee15bc36dc6c) --- .../pagemem/wal/record/LazyDataEntry.java | 18 ++++---- .../pagemem/wal/record/LazyMvccDataEntry.java | 18 ++++---- .../wal/record/MarshalledDataEntry.java | 45 +++++++++++++++++++ .../pagemem/wal/record/UnwrapDataEntry.java | 20 +++------ .../wal/record/UnwrapMvccDataEntry.java | 20 +++------ .../wal/record/UnwrappedDataEntry.java | 39 ++++++++++++++++ .../reader/StandaloneWalRecordsIterator.java | 10 ++--- .../db/wal/reader/IgniteWalReaderTest.java | 19 +++----- 8 files changed, 123 insertions(+), 66 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MarshalledDataEntry.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrappedDataEntry.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java index 403d77889e292..ba2fabc9f0eeb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java @@ -31,7 +31,7 @@ * Represents Data Entry ({@link #key}, {@link #val value}) pair update {@link #op operation}.
* This Data entry was not converted to key, value pair during record deserialization. */ -public class LazyDataEntry extends DataEntry { +public class LazyDataEntry extends DataEntry implements MarshalledDataEntry { /** */ private GridCacheSharedContext cctx; @@ -124,23 +124,23 @@ public LazyDataEntry( return val; } - /** @return Data Entry Key type code. See {@link CacheObject} for built-in value type codes */ - public byte getKeyType() { + /** {@inheritDoc} */ + @Override public byte getKeyType() { return keyType; } - /** @return Key value bytes. */ - public byte[] getKeyBytes() { + /** {@inheritDoc} */ + @Override public byte[] getKeyBytes() { return keyBytes; } - /** @return Data Entry Value type code. See {@link CacheObject} for built-in value type codes */ - public byte getValType() { + /** {@inheritDoc} */ + @Override public byte getValType() { return valType; } - /** @return Value value bytes. */ - public byte[] getValBytes() { + /** {@inheritDoc} */ + @Override public byte[] getValBytes() { return valBytes; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyMvccDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyMvccDataEntry.java index 15b146807dc44..a7ad86f4eaf82 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyMvccDataEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyMvccDataEntry.java @@ -32,7 +32,7 @@ * Represents Data Entry ({@link #key}, {@link #val value}) pair update {@link #op operation}.
* This Data entry was not converted to key, value pair during record deserialization. */ -public class LazyMvccDataEntry extends MvccDataEntry { +public class LazyMvccDataEntry extends MvccDataEntry implements MarshalledDataEntry { /** */ private GridCacheSharedContext cctx; @@ -127,23 +127,23 @@ public LazyMvccDataEntry( return val; } - /** @return Data Entry Key type code. See {@link CacheObject} for built-in value type codes */ - public byte getKeyType() { + /** {@inheritDoc} */ + @Override public byte getKeyType() { return keyType; } - /** @return Key value bytes. */ - public byte[] getKeyBytes() { + /** {@inheritDoc} */ + @Override public byte[] getKeyBytes() { return keyBytes; } - /** @return Data Entry Value type code. See {@link CacheObject} for built-in value type codes */ - public byte getValType() { + /** {@inheritDoc} */ + @Override public byte getValType() { return valType; } - /** @return Value value bytes. */ - public byte[] getValBytes() { + /** {@inheritDoc} */ + @Override public byte[] getValBytes() { return valBytes; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MarshalledDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MarshalledDataEntry.java new file mode 100644 index 0000000000000..c977d527941e9 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MarshalledDataEntry.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.pagemem.wal.record; + +import org.apache.ignite.internal.processors.cache.CacheObject; + +/** + * Interface for Data Entry record that was not converted to key, value pair during record deserialization. + */ +public interface MarshalledDataEntry { + /** + * @return Data Entry Key type code. See {@link CacheObject} for built-in value type codes. + */ + byte getKeyType(); + + /** + * @return Key value bytes. + */ + byte[] getKeyBytes(); + + /** + * @return Data Entry Value type code. See {@link CacheObject} for built-in value type codes. + */ + byte getValType(); + + /** + * @return Value value bytes. + */ + byte[] getValBytes(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java index dbcc65176b6fa..5dd268b3b8ca8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java @@ -27,7 +27,7 @@ /** * Data Entry for automatic unwrapping key and value from Data Entry */ -public class UnwrapDataEntry extends DataEntry { +public class UnwrapDataEntry extends DataEntry implements UnwrappedDataEntry { /** Cache object value context. Context is used for unwrapping objects. */ private final CacheObjectValueContext cacheObjValCtx; @@ -64,13 +64,8 @@ public UnwrapDataEntry( this.keepBinary = keepBinary; } - /** - * Unwraps key value from cache key object into primitive boxed type or source class. If client classes were used - * in key, call of this method requires classes to be available in classpath. - * - * @return Key which was placed into cache. Or null if failed to convert. - */ - public Object unwrappedKey() { + /** {@inheritDoc} */ + @Override public Object unwrappedKey() { try { if (keepBinary && key instanceof BinaryObject) return key; @@ -93,13 +88,8 @@ public Object unwrappedKey() { } } - /** - * Unwraps value value from cache value object into primitive boxed type or source class. If client classes were - * used in key, call of this method requires classes to be available in classpath. - * - * @return Value which was placed into cache. Or null for delete operation or for failure. - */ - public Object unwrappedValue() { + /** {@inheritDoc} */ + @Override public Object unwrappedValue() { try { if (val == null) return null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java index 25d7a6e3f947b..c3c12a372e757 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java @@ -28,7 +28,7 @@ /** * Data Entry for automatic unwrapping key and value from Mvcc Data Entry */ -public class UnwrapMvccDataEntry extends MvccDataEntry { +public class UnwrapMvccDataEntry extends MvccDataEntry implements UnwrappedDataEntry { /** Cache object value context. Context is used for unwrapping objects. */ private final CacheObjectValueContext cacheObjValCtx; @@ -68,13 +68,8 @@ public UnwrapMvccDataEntry( this.keepBinary = keepBinary; } - /** - * Unwraps key value from cache key object into primitive boxed type or source class. If client classes were used - * in key, call of this method requires classes to be available in classpath. - * - * @return Key which was placed into cache. Or null if failed to convert. - */ - public Object unwrappedKey() { + /** {@inheritDoc} */ + @Override public Object unwrappedKey() { try { if (keepBinary && key instanceof BinaryObject) return key; @@ -97,13 +92,8 @@ public Object unwrappedKey() { } } - /** - * Unwraps value value from cache value object into primitive boxed type or source class. If client classes were - * used in key, call of this method requires classes to be available in classpath. - * - * @return Value which was placed into cache. Or null for delete operation or for failure. - */ - public Object unwrappedValue() { + /** {@inheritDoc} */ + @Override public Object unwrappedValue() { try { if (val == null) return null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrappedDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrappedDataEntry.java new file mode 100644 index 0000000000000..b3a20b9ca0525 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrappedDataEntry.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.pagemem.wal.record; + +/** + * Interface for Data Entry for automatic unwrapping key and value from Data Entry + */ +public interface UnwrappedDataEntry { + /** + * Unwraps key value from cache key object into primitive boxed type or source class. If client classes were used in + * key, call of this method requires classes to be available in classpath. + * + * @return Key which was placed into cache. Or null if failed to convert. + */ + Object unwrappedKey(); + + /** + * Unwraps value value from cache value object into primitive boxed type or source class. If client classes were + * used in key, call of this method requires classes to be available in classpath. + * + * @return Value which was placed into cache. Or null for delete operation or for failure. + */ + Object unwrappedValue(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index 00495e135e5a8..56a44cd8ad4e3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -29,7 +29,7 @@ import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; import org.apache.ignite.internal.pagemem.wal.record.FilteredRecord; -import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry; +import org.apache.ignite.internal.pagemem.wal.record.MarshalledDataEntry; import org.apache.ignite.internal.pagemem.wal.record.MvccDataEntry; import org.apache.ignite.internal.pagemem.wal.record.MvccDataRecord; import org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry; @@ -434,15 +434,15 @@ private boolean checkBounds(long idx) { final IgniteCacheObjectProcessor processor, final CacheObjectContext fakeCacheObjCtx, final DataEntry dataEntry) throws IgniteCheckedException { - if(dataEntry instanceof EncryptedDataEntry) + if (dataEntry instanceof EncryptedDataEntry) return dataEntry; final KeyCacheObject key; final CacheObject val; boolean keepBinary = this.keepBinary || !fakeCacheObjCtx.kernalContext().marshallerContext().initialized(); - if (dataEntry instanceof LazyDataEntry) { - final LazyDataEntry lazyDataEntry = (LazyDataEntry)dataEntry; + if (dataEntry instanceof MarshalledDataEntry) { + final MarshalledDataEntry lazyDataEntry = (MarshalledDataEntry)dataEntry; key = processor.toKeyCacheObject(fakeCacheObjCtx, lazyDataEntry.getKeyType(), @@ -472,7 +472,7 @@ private boolean checkBounds(long idx) { * @param keepBinary Don't convert non primitive types. * @return Unwrapped entry. */ - private @NotNull DataEntry unwrapDataEntry(CacheObjectContext coCtx, DataEntry dataEntry, + private DataEntry unwrapDataEntry(CacheObjectContext coCtx, DataEntry dataEntry, KeyCacheObject key, CacheObject val, boolean keepBinary) { if (dataEntry instanceof MvccDataEntry) return new UnwrapMvccDataEntry( diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index f6f8a953820f0..bcd3589795c45 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -61,9 +61,10 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; -import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry; +import org.apache.ignite.internal.pagemem.wal.record.MarshalledDataEntry; import org.apache.ignite.internal.pagemem.wal.record.TxRecord; import org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry; +import org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.GridCacheOperation; @@ -125,9 +126,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { /** Custom wal mode. */ private WALMode customWalMode; - /** Clear properties in afterTest() method. */ - private boolean clearProps; - /** Set WAL and Archive path to same value. */ private boolean setWalAndArchiveToSameVal; @@ -186,8 +184,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - stopAllGrids(); - cleanPersistenceDir(); } @@ -197,8 +193,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { cleanPersistenceDir(); - if (clearProps) - System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS); + System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS); } /** @@ -1096,8 +1091,6 @@ public void testPutAllTxIntoTwoNodes() throws Exception { * @throws Exception if failed. */ public void testTxRecordsReadWoBinaryMeta() throws Exception { - clearProps = true; - System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true"); Ignite ignite = startGrid("node0"); @@ -1356,12 +1349,12 @@ private Map iterateAndCountDataRecord( Object unwrappedKeyObj; Object unwrappedValObj; - if (entry instanceof UnwrapDataEntry) { - UnwrapDataEntry unwrapDataEntry = (UnwrapDataEntry)entry; + if (entry instanceof UnwrappedDataEntry) { + UnwrappedDataEntry unwrapDataEntry = (UnwrappedDataEntry)entry; unwrappedKeyObj = unwrapDataEntry.unwrappedKey(); unwrappedValObj = unwrapDataEntry.unwrappedValue(); } - else if (entry instanceof LazyDataEntry) { + else if (entry instanceof MarshalledDataEntry) { unwrappedKeyObj = null; unwrappedValObj = null; //can't check value From 1476abe166cb17b59c72f2bf59844f66068b1ce1 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Wed, 19 Dec 2018 14:44:16 +0300 Subject: [PATCH 272/403] IGNITE-10572: MVCC TX: Possible race on invokeAll operations. This closes #5608. (cherry picked from commit 9dc6d0864fb007fcab97eccee964d1df75fbd75c) --- .../near/GridNearTxEnlistFuture.java | 20 ++++++++++--------- .../near/GridNearTxEnlistResponse.java | 2 +- .../distributed/near/GridNearTxLocal.java | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java index 598ffc353134f..3b72294fe2d5e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java @@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; @@ -70,6 +71,10 @@ public class GridNearTxEnlistFuture extends GridNearTxAbstractEnlistFuture SKIP_UPD = AtomicIntegerFieldUpdater.newUpdater(GridNearTxEnlistFuture.class, "skipCntr"); + /** Res field updater. */ + private static final AtomicReferenceFieldUpdater RES_UPD = + AtomicReferenceFieldUpdater.newUpdater(GridNearTxEnlistFuture.class, GridCacheReturn.class, "res"); + /** Marker object. */ private static final Object FINISHED = new Object(); @@ -599,20 +604,17 @@ public boolean checkResponse(UUID nodeId, GridNearTxEnlistResponse res, Throwabl assert res != null; - if (res.result().invokeResult()) { - if(this.res == null) - this.res = new GridCacheReturn(true, true); - - this.res.success(this.res.success() && err == null && res.result().success()); + if (this.res != null || !RES_UPD.compareAndSet(this, null, res.result())) { + GridCacheReturn res0 = this.res; - this.res.mergeEntryProcessResults(res.result()); + if (res.result().invokeResult()) + res0.mergeEntryProcessResults(res.result()); + else if (res0.success() && !res.result().success()) + res0.success(false); } - else - this.res = res.result(); assert this.res != null && (this.res.emptyResult() || needRes || this.res.invokeResult() || !this.res.success()); - tx.hasRemoteLocks(true); return true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java index cf00186eb1213..93df0c2d78fea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java @@ -158,7 +158,7 @@ public GridCacheReturn result() { } /** {@inheritDoc} */ - @Nullable @Override public Throwable error() { + @Override public @Nullable Throwable error() { return err; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java index 09c560181862f..7a2a7eeb97f21 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java @@ -2093,7 +2093,7 @@ private IgniteInternalFuture updateAsync(GridCacheContext cache Object val = futRes.value(); - if (futRes.invokeResult()) { + if (futRes.invokeResult() && val != null) { assert val instanceof Map; val = cacheCtx.unwrapInvokeResult((Map)val, keepBinary); From 5870a0514ccdc832f00c98d13b9c1e1292cbaaa8 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Mon, 10 Dec 2018 14:35:08 +0300 Subject: [PATCH 273/403] IGNITE-10366: MVCC: Create "Cache 1" test suite for MVCC mode. This closes #5500. (cherry picked from commit 2b2f50ca00a48ec94b5f760e324e60a0c33e5713) --- ...CacheEntryProcessorSequentialCallTest.java | 68 ++- ...eTransactionalStoreReadFromBackupTest.java | 8 + ...GridCacheLoadOnlyStoreAdapterSelfTest.java | 81 ++-- .../CacheJdbcPojoStoreAbstractSelfTest.java | 10 +- ...cheJdbcBlobStoreMultithreadedSelfTest.java | 8 + .../IgniteDiagnosticMessagesTest.java | 142 +++++-- .../cache/CacheDeferredDeleteQueueTest.java | 12 + .../CacheDeferredDeleteSanitySelfTest.java | 29 ++ .../cache/CacheFutureExceptionSelfTest.java | 6 + .../cache/CacheMvccTxFastFinishTest.java | 67 +++ .../CacheNamesWithSpecialCharactersTest.java | 9 +- .../CachePutEventListenerErrorSelfTest.java | 3 + .../cache/CacheTxFastFinishTest.java | 8 +- ...dCacheAbstractByteArrayValuesSelfTest.java | 6 - .../cache/GridCacheClearAllSelfTest.java | 9 + ...CacheConfigurationConsistencySelfTest.java | 27 ++ .../GridCacheEntryMemorySizeSelfTest.java | 29 +- ...ridCacheLocalTxStoreExceptionSelfTest.java | 8 + ...idCacheMixedPartitionExchangeSelfTest.java | 9 + ...dCacheMvccMultiThreadedUpdateSelfTest.java | 203 +++++++++ .../GridCacheObjectToStringSelfTest.java | 8 + .../cache/GridCacheP2PUndeploySelfTest.java | 3 - .../cache/GridCacheStopSelfTest.java | 78 +++- .../cache/GridCacheStorePutxSelfTest.java | 159 ------- .../GridCacheTtlManagerEvictionSelfTest.java | 9 + .../cache/GridCacheTtlManagerSelfTest.java | 8 + ...dCacheTxPartitionedLocalStoreSelfTest.java | 8 + .../IgniteCacheEntryListenerAbstractTest.java | 8 + ...acheEntryListenerEagerTtlDisabledTest.java | 8 + .../IgniteCacheEntryListenerTxLocalTest.java | 8 + .../IgniteCacheEntryProcessorCallTest.java | 53 ++- .../cache/IgniteCacheInvokeAbstractTest.java | 13 +- ....java => IgniteCacheMvccTxInvokeTest.java} | 23 +- ...gniteCacheMvccTxNearEnabledInvokeTest.java | 37 ++ .../IgniteCacheNearLockValueSelfTest.java | 3 + .../cache/IgniteCacheTxLocalInvokeTest.java | 8 + .../cache/IgnitePutAllLargeBatchSelfTest.java | 8 + ...llUpdateNonPreloadedPartitionSelfTest.java | 8 + .../cache/IgniteTxConfigCacheSelfTest.java | 8 +- .../IgniteTxExceptionAbstractSelfTest.java | 18 +- ...gniteTxStoreExceptionAbstractSelfTest.java | 3 + .../CacheKeepBinaryWithInterceptorTest.java | 40 +- ...eamProcessorPersistenceBinarySelfTest.java | 28 ++ .../GridDataStreamerImplSelfTest.java | 2 +- .../CacheTxNearUpdateTopologyChangeTest.java | 8 + ...actDistributedByteArrayValuesSelfTest.java | 65 ++- ...actPartitionedByteArrayValuesSelfTest.java | 27 +- .../IgniteCacheConnectionRecoveryTest.java | 15 +- ...acheMessageRecoveryIdleConnectionTest.java | 8 + ...IgniteCacheSystemTransactionsSelfTest.java | 45 +- .../IgniteCrossCacheTxStoreSelfTest.java | 11 +- ...artitionedOnlyByteArrayValuesSelfTest.java | 17 +- .../dht/GridCacheGlobalLoadTest.java | 8 + ...earPartitionedByteArrayValuesSelfTest.java | 14 + .../near/GridCacheGetStoreErrorSelfTest.java | 11 +- .../GridCacheNearTxExceptionSelfTest.java | 8 + .../GridCachePartitionedStorePutSelfTest.java | 58 ++- ...ractReplicatedByteArrayValuesSelfTest.java | 2 +- ...GridCacheLocalByteArrayValuesSelfTest.java | 5 +- .../GridCacheLocalTxExceptionSelfTest.java | 8 + ...ntryProcessorExternalizableFailedTest.java | 392 +++++++++++++----- ...acheEntryProcessorNonSerializableTest.java | 240 ++++++++--- .../TxRollbackOnTopologyChangeTest.java | 4 + .../DataStreamProcessorMvccSelfTest.java | 2 +- .../DataStreamProcessorSelfTest.java | 45 +- .../testsuites/IgniteBasicTestSuite.java | 2 + .../IgniteBinaryCacheTestSuite.java | 48 ++- .../testsuites/IgniteCacheMvccTestSuite1.java | 247 +++++++++++ ...gniteCacheTcpClientDiscoveryTestSuite.java | 41 +- .../testsuites/IgniteCacheTestSuite.java | 330 +++++++-------- .../testsuites/IgniteCacheTestSuite2.java | 1 - 71 files changed, 2141 insertions(+), 812 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMvccTxFastFinishTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccMultiThreadedUpdateSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStorePutxSelfTest.java rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/{context/IgniteCachePartitionedExecutionContextTest.java => IgniteCacheMvccTxInvokeTest.java} (65%) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxNearEnabledInvokeTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/DataStreamProcessorPersistenceBinarySelfTest.java create mode 100755 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite1.java diff --git a/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java b/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java index 165bca78b2aa8..f94085653c079 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java @@ -19,6 +19,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.MutableEntry; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; @@ -28,13 +30,20 @@ import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; -import javax.cache.processor.EntryProcessorException; -import javax.cache.processor.MutableEntry; import org.apache.ignite.transactions.TransactionOptimisticException; /** */ public class IgniteCacheEntryProcessorSequentialCallTest extends GridCommonAbstractTest { + /** */ + private static final String CACHE = "cache"; + + /** */ + private static final String MVCC_CACHE = "mvccCache"; + + /** */ + private String cacheName; + /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { startGrids(2); @@ -45,11 +54,33 @@ public class IgniteCacheEntryProcessorSequentialCallTest extends GridCommonAbstr stopAllGrids(); } + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cacheName = CACHE; + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - CacheConfiguration cacheCfg = new CacheConfiguration("cache"); + CacheConfiguration ccfg = cacheConfiguration(CACHE); + + CacheConfiguration mvccCfg = cacheConfiguration(MVCC_CACHE) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + + cfg.setCacheConfiguration(ccfg, mvccCfg); + + return cfg; + } + + /** + * + * @return Cache configuration. + * @param name Cache name. + */ + private CacheConfiguration cacheConfiguration(String name) { + CacheConfiguration cacheCfg = new CacheConfiguration(name); cacheCfg.setCacheMode(CacheMode.PARTITIONED); cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); @@ -57,10 +88,7 @@ public class IgniteCacheEntryProcessorSequentialCallTest extends GridCommonAbstr cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); cacheCfg.setMaxConcurrentAsyncOperations(0); cacheCfg.setBackups(0); - - cfg.setCacheConfiguration(cacheCfg); - - return cfg; + return cacheCfg; } /** @@ -117,6 +145,17 @@ public void testPessimisticReadCommittedTxInvokeSequentialCall() throws Exceptio transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED); } + /** + * + */ + public void testMvccTxInvokeSequentialCall() throws Exception { + cacheName = MVCC_CACHE; + + transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + + transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + } + /** * Test for sequential entry processor invoking not null value on primary cache. * In this test entry processor gets value from local node. @@ -132,12 +171,12 @@ public void transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency Ignite primaryIgnite; - if (ignite(0).affinity("cache").isPrimary(ignite(0).cluster().localNode(), key)) + if (ignite(0).affinity(cacheName).isPrimary(ignite(0).cluster().localNode(), key)) primaryIgnite = ignite(0); else primaryIgnite = ignite(1); - IgniteCache cache = primaryIgnite.cache("cache"); + IgniteCache cache = primaryIgnite.cache(cacheName); cache.put(key, val); @@ -171,7 +210,7 @@ public void transactionInvokeSequentialCallOnNearNode(TransactionConcurrency tra Ignite nearIgnite; Ignite primaryIgnite; - if (ignite(0).affinity("cache").isPrimary(ignite(0).cluster().localNode(), key)) { + if (ignite(0).affinity(cacheName).isPrimary(ignite(0).cluster().localNode(), key)) { primaryIgnite = ignite(0); nearIgnite = ignite(1); @@ -182,9 +221,9 @@ public void transactionInvokeSequentialCallOnNearNode(TransactionConcurrency tra nearIgnite = ignite(0); } - primaryIgnite.cache("cache").put(key, val); + primaryIgnite.cache(cacheName).put(key, val); - IgniteCache nearCache = nearIgnite.cache("cache"); + IgniteCache nearCache = nearIgnite.cache(cacheName); NotNullCacheEntryProcessor cacheEntryProcessor = new NotNullCacheEntryProcessor(); @@ -197,17 +236,18 @@ public void transactionInvokeSequentialCallOnNearNode(TransactionConcurrency tra transaction.commit(); } - primaryIgnite.cache("cache").remove(key); + primaryIgnite.cache(cacheName).remove(key); } /** * Test for sequential entry processor invocation. During transaction value is changed externally, which leads to * optimistic conflict exception. */ + @SuppressWarnings("ThrowableNotThrown") public void testTxInvokeSequentialOptimisticConflict() throws Exception { TestKey key = new TestKey(1L); - IgniteCache cache = ignite(0).cache("cache"); + IgniteCache cache = ignite(0).cache(CACHE); CountDownLatch latch = new CountDownLatch(1); diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java index 4837936621f46..f1b14aba08988 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheTransactionalStoreReadFromBackupTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.cache.store; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -25,6 +26,13 @@ * */ public class CacheTransactionalStoreReadFromBackupTest extends CacheStoreReadFromBackupTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/GridCacheLoadOnlyStoreAdapterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/GridCacheLoadOnlyStoreAdapterSelfTest.java index 2a1e23a8bcbd7..6273d6fc0b4cd 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/GridCacheLoadOnlyStoreAdapterSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/GridCacheLoadOnlyStoreAdapterSelfTest.java @@ -21,51 +21,51 @@ import java.util.Iterator; import java.util.NoSuchElementException; import javax.cache.integration.CacheLoaderException; +import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.testframework.MvccFeatureChecker; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; /** * */ -public class GridCacheLoadOnlyStoreAdapterSelfTest extends GridCacheAbstractSelfTest { +public class GridCacheLoadOnlyStoreAdapterSelfTest extends GridCommonAbstractTest { /** Expected loadAll arguments, hardcoded on call site for convenience. */ private static final Integer[] EXP_ARGS = {1, 2, 3}; /** Store to use. */ private CacheLoadOnlyStoreAdapter store; - /** {@inheritDoc} */ - @Override protected int gridCount() { - return 1; - } - /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); - } - - /** {@inheritDoc} */ - @Override protected void afterTestsStopped() throws Exception { + super.beforeTestsStarted(); + startGrid(0); } /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); } /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + super.afterTest(); } - /** {@inheritDoc} */ + /** + * @return Cache configuration. + */ @SuppressWarnings("unchecked") - @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { - CacheConfiguration cfg = super.cacheConfiguration(igniteInstanceName); + private CacheConfiguration cacheConfiguration() { + CacheConfiguration cfg = defaultCacheConfiguration(); assertNotNull(store); @@ -81,58 +81,35 @@ public class GridCacheLoadOnlyStoreAdapterSelfTest extends GridCacheAbstractSelf * @throws Exception If failed. */ public void testStore() throws Exception { - try { - int inputSize = 100; - - store = new TestStore(inputSize); - - startGrids(gridCount()); + int inputSize = 100; - awaitPartitionMapExchange(); + store = new TestStore(inputSize); - jcache().localLoadCache(null, 1, 2, 3); + IgniteCache cache = grid(0).createCache(cacheConfiguration()); - int cnt = 0; + cache.localLoadCache(null, 1, 2, 3); - for (int i = 0; i < gridCount(); i++) - cnt += jcache(i).localSize(); - - assertEquals(inputSize - (inputSize / 10), cnt); - } - finally { - stopAllGrids(); - } + assertEquals(inputSize - (inputSize / 10), cache.localSize()); } /** * @throws Exception If failed. */ public void testStoreSmallQueueSize() throws Exception { - try { - int inputSize = 1500; - - store = new ParallelTestStore(inputSize); - - store.setBatchSize(1); - store.setBatchQueueSize(1); - store.setThreadsCount(2); + int inputSize = 1500; - startGrids(gridCount()); + store = new ParallelTestStore(inputSize); - awaitPartitionMapExchange(); + store.setBatchSize(1); + store.setBatchQueueSize(1); + store.setThreadsCount(2); - jcache().localLoadCache(null, 1, 2, 3); + IgniteCache cache = grid(0).createCache(cacheConfiguration()); - int cnt = 0; + cache.localLoadCache(null, 1, 2, 3); - for (int i = 0; i < gridCount(); i++) - cnt += jcache(i).localSize(); + assertEquals(inputSize, cache.localSize()); - assertEquals(inputSize, cnt); - } - finally { - stopAllGrids(); - } } /** diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java index 703cbe18908b9..20958245c0e0e 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreAbstractSelfTest.java @@ -29,8 +29,8 @@ import javax.cache.integration.CacheLoaderException; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect; -import org.apache.ignite.cache.store.jdbc.model.Person; import org.apache.ignite.cache.store.jdbc.model.Gender; +import org.apache.ignite.cache.store.jdbc.model.Person; import org.apache.ignite.cache.store.jdbc.model.PersonKey; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.ConnectorConfiguration; @@ -41,6 +41,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; @@ -96,6 +97,13 @@ protected Connection getConnection() throws SQLException { return DriverManager.getConnection(DFLT_CONN_URL, "sa", ""); } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { Connection conn = getConnection(); diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/GridCacheJdbcBlobStoreMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/GridCacheJdbcBlobStoreMultithreadedSelfTest.java index 1f6849030a664..9eaedbf9f7286 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/GridCacheJdbcBlobStoreMultithreadedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/GridCacheJdbcBlobStoreMultithreadedSelfTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -65,6 +66,13 @@ public class GridCacheJdbcBlobStoreMultithreadedSelfTest extends GridCommonAbstr /** Client flag. */ private boolean client; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { startGridsMultiThreaded(GRID_CNT - 2); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java index a29d572cb6949..a7dc96cc3f88a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -48,12 +49,13 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridStringLogger; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; @@ -110,7 +112,14 @@ public class IgniteDiagnosticMessagesTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testDiagnosticMessages1() throws Exception { - checkBasicDiagnosticInfo(); + checkBasicDiagnosticInfo(CacheAtomicityMode.TRANSACTIONAL); + } + + /** + * @throws Exception If failed. + */ + public void testDiagnosticMessagesMvcc1() throws Exception { + checkBasicDiagnosticInfo(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); } /** @@ -119,13 +128,37 @@ public void testDiagnosticMessages1() throws Exception { public void testDiagnosticMessages2() throws Exception { connectionsPerNode = 5; - checkBasicDiagnosticInfo(); + checkBasicDiagnosticInfo(CacheAtomicityMode.TRANSACTIONAL); + } + + /** + * @throws Exception If failed. + */ + public void testDiagnosticMessagesMvcc2() throws Exception { + connectionsPerNode = 5; + + checkBasicDiagnosticInfo(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); } /** * @throws Exception If failed. */ public void testLongRunning() throws Exception { + checkLongRunning(TRANSACTIONAL); + } + + /** + * @throws Exception If failed. + */ + public void testLongRunningMvcc() throws Exception { + checkLongRunning(TRANSACTIONAL_SNAPSHOT); + } + + /** + * @param atomicityMode Cache atomicity mode. + * @throws Exception If failed. + */ + public void checkLongRunning(CacheAtomicityMode atomicityMode) throws Exception { System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, "3500"); try { @@ -139,11 +172,7 @@ public void testLongRunning() throws Exception { awaitPartitionMapExchange(); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setCacheMode(PARTITIONED); - ccfg.setAtomicityMode(TRANSACTIONAL); + CacheConfiguration ccfg = cacheConfiguration(atomicityMode); final Ignite node0 = ignite(0); @@ -183,10 +212,39 @@ public void testLongRunning() throws Exception { } } + /** + * @param atomicityMode Cache atomicity mode. + * @return Cache configuration. + */ + @SuppressWarnings("unchecked") + private CacheConfiguration cacheConfiguration(CacheAtomicityMode atomicityMode) { + return defaultCacheConfiguration() + .setAtomicityMode(atomicityMode) + .setWriteSynchronizationMode(FULL_SYNC) + .setNearConfiguration(null); + } + + /** + * @throws Exception If failed. + */ + public void testSeveralLongRunningMvccTxs() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9322"); // Fix diagnostic message or disable test. + + checkSeveralLongRunningTxs(TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ public void testSeveralLongRunningTxs() throws Exception { + checkSeveralLongRunningTxs(TRANSACTIONAL); + } + + /** + * @param atomicityMode Cache atomicity mode. + * @throws Exception If failed. + */ + public void checkSeveralLongRunningTxs(CacheAtomicityMode atomicityMode) throws Exception { int timeout = 3500; System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, String.valueOf(timeout)); @@ -204,11 +262,7 @@ public void testSeveralLongRunningTxs() throws Exception { awaitPartitionMapExchange(); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setCacheMode(PARTITIONED); - ccfg.setAtomicityMode(TRANSACTIONAL); + CacheConfiguration ccfg = cacheConfiguration(atomicityMode); final Ignite node0 = ignite(0); final Ignite node1 = ignite(1); @@ -291,11 +345,31 @@ private int countTxKeysInASingleBlock(String log) { return cnt; } + /** + * @throws Exception If failed. + */ + public void testLongRunningMvccTx() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9322"); // Fix diagnostic message or disable test. + + checkLongRunningTx(TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ public void testLongRunningTx() throws Exception { - System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, "3500"); + checkLongRunningTx(TRANSACTIONAL); + + } + + /** + * @param atomicityMode Cache atomicity mode. + * @throws Exception If failed. + */ + public void checkLongRunningTx(CacheAtomicityMode atomicityMode) throws Exception { + final int longOpDumpTimeout = 1000; + + System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, String.valueOf(longOpDumpTimeout)); try { startGrid(0); @@ -308,11 +382,7 @@ public void testLongRunningTx() throws Exception { awaitPartitionMapExchange(); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setCacheMode(PARTITIONED); - ccfg.setAtomicityMode(TRANSACTIONAL); + CacheConfiguration ccfg = cacheConfiguration(atomicityMode); final Ignite node0 = ignite(0); final Ignite node1 = ignite(1); @@ -387,6 +457,21 @@ public void testLongRunningTx() throws Exception { * @throws Exception If failed. */ public void testRemoteTx() throws Exception { + checkRemoteTx(TRANSACTIONAL); + } + + /** + * @throws Exception If failed. + */ + public void testRemoteMvccTx() throws Exception { + checkRemoteTx(TRANSACTIONAL_SNAPSHOT); + } + + /** + * @param atomicityMode Cache atomicity mode. + * @throws Exception If failed. + */ + public void checkRemoteTx(CacheAtomicityMode atomicityMode) throws Exception { int timeout = 3500; System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, String.valueOf(timeout)); @@ -404,13 +489,11 @@ public void testRemoteTx() throws Exception { awaitPartitionMapExchange(); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + CacheConfiguration ccfg = cacheConfiguration(atomicityMode).setBackups(1); - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setCacheMode(PARTITIONED); - ccfg.setAtomicityMode(TRANSACTIONAL); - ccfg.setBackups(1); - ccfg.setNearConfiguration(new NearCacheConfiguration()); + if (atomicityMode != TRANSACTIONAL_SNAPSHOT || + MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) + ccfg.setNearConfiguration(new NearCacheConfiguration<>()); final Ignite node0 = ignite(0); final Ignite node1 = ignite(1); @@ -462,9 +545,10 @@ public void testRemoteTx() throws Exception { } /** + * @param atomicityMode Cache atomicity mode. * @throws Exception If failed. */ - private void checkBasicDiagnosticInfo() throws Exception { + private void checkBasicDiagnosticInfo(CacheAtomicityMode atomicityMode) throws Exception { startGrids(3); client = true; @@ -473,11 +557,7 @@ private void checkBasicDiagnosticInfo() throws Exception { startGrid(4); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setCacheMode(REPLICATED); - ccfg.setAtomicityMode(TRANSACTIONAL); + CacheConfiguration ccfg = cacheConfiguration(atomicityMode).setCacheMode(REPLICATED); ignite(0).createCache(ccfg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java index 871bc338e9698..479ac8ecb5fe0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java @@ -32,6 +32,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -70,11 +71,22 @@ public void testDeferredDeleteQueue() throws Exception { testQueue(TRANSACTIONAL, false); + testQueue(TRANSACTIONAL_SNAPSHOT, false); + testQueue(ATOMIC, true); testQueue(TRANSACTIONAL, true); } + /** + * @throws Exception If failed. + */ + public void testDeferredDeleteQueueMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + testQueue(TRANSACTIONAL_SNAPSHOT, true); + } + /** * @param atomicityMode Cache atomicity mode. * @param nearCache {@code True} if need create near cache. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java index dd7579905b29d..603938437516f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java @@ -26,6 +26,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -63,6 +64,34 @@ public void testDeferredDelete() throws Exception { testDeferredDelete(REPLICATED, TRANSACTIONAL, true, true); } + /** + * @throws Exception If fails. + */ + public void testDeferredDeleteMvcc() throws Exception { + testDeferredDelete(PARTITIONED, TRANSACTIONAL_SNAPSHOT, false, true); + testDeferredDelete(REPLICATED, TRANSACTIONAL_SNAPSHOT, false, true); + } + + /** + * @throws Exception If fails. + */ + public void testDeferredDeleteMvccNear() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + testDeferredDelete(PARTITIONED, TRANSACTIONAL_SNAPSHOT, true, false); + testDeferredDelete(REPLICATED, TRANSACTIONAL_SNAPSHOT, true, true); + } + + /** + * @throws Exception If fails. + */ + public void testDeferredDeleteMvccLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + testDeferredDelete(LOCAL, TRANSACTIONAL_SNAPSHOT, false, false); + testDeferredDelete(LOCAL, TRANSACTIONAL_SNAPSHOT, true, false); + } + /** * @param mode Mode. * @param atomicityMode Atomicity mode. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheFutureExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheFutureExceptionSelfTest.java index a51765c652d4c..78cd568b1d3c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheFutureExceptionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheFutureExceptionSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.concurrent.TimeUnit.SECONDS; @@ -93,6 +94,11 @@ public void testAsyncCacheFuture() throws Exception { * @throws Exception If failed. */ private void testGet(boolean nearCache, boolean cpyOnRead) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + if (!MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) + return; + } + fail = false; Ignite srv = grid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMvccTxFastFinishTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMvccTxFastFinishTest.java new file mode 100644 index 0000000000000..7261d79162a30 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMvccTxFastFinishTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteTransactions; +import org.apache.ignite.transactions.Transaction; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * + */ +public class CacheMvccTxFastFinishTest extends CacheTxFastFinishTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10444"); + + super.beforeTestsStarted(); + } + + /** {@inheritDoc} */ + @Override protected void fastFinishTx(Ignite ignite) { + IgniteTransactions txs = ignite.transactions(); + + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); + + for (boolean commit : new boolean[] {true, false}) { + try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) { + checkFastTxFinish(tx, commit); + } + + for (int i = 0; i < 100; i++) { + try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.get(i); + + checkNormalTxFinish(tx, commit); + } + } + + for (int i = 0; i < 100; i++) { + try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.put(i, i); + + checkNormalTxFinish(tx, commit); + } + } + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNamesWithSpecialCharactersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNamesWithSpecialCharactersTest.java index ecb8227e58f53..96b7c0a788779 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNamesWithSpecialCharactersTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNamesWithSpecialCharactersTest.java @@ -17,21 +17,22 @@ package org.apache.ignite.internal.processors.cache; +import java.util.Collection; import org.apache.ignite.Ignite; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import java.util.Collection; - /** * Test that validates {@link Ignite#cacheNames()} implementation. */ public class CacheNamesWithSpecialCharactersTest extends GridCommonAbstractTest { + /** */ + private static final String CACHE_NAME_1 = "--№=+:(replicated)"; - public static final String CACHE_NAME_1 = "--№=+:(replicated)"; - public static final String CACHE_NAME_2 = ":_&:: (partitioned)"; + /** */ + private static final String CACHE_NAME_2 = ":_&:: (partitioned)"; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutEventListenerErrorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutEventListenerErrorSelfTest.java index 8891f64233476..a6060ded84304 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutEventListenerErrorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutEventListenerErrorSelfTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -60,6 +61,8 @@ public class CachePutEventListenerErrorSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + startGridsMultiThreaded(3); Ignition.setClientMode(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java index 79316bfa457b4..b9a41ba9d4146 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java @@ -107,7 +107,7 @@ public void testFastFinishTx() throws Exception { /** * @throws Exception If failed. */ - private void fastFinishTx() throws Exception { + protected void fastFinishTx() throws Exception { startGrid(0); fastFinishTx(ignite(0)); @@ -142,7 +142,7 @@ private void fastFinishTx() throws Exception { /** * @param ignite Node. */ - private void fastFinishTx(Ignite ignite) { + protected void fastFinishTx(Ignite ignite) { IgniteTransactions txs = ignite.transactions(); IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); @@ -202,7 +202,7 @@ private void fastFinishTx(Ignite ignite) { * @param tx Transaction. * @param commit Commit flag. */ - private void checkFastTxFinish(Transaction tx, boolean commit) { + protected void checkFastTxFinish(Transaction tx, boolean commit) { if (commit) tx.commit(); else @@ -218,7 +218,7 @@ private void checkFastTxFinish(Transaction tx, boolean commit) { * @param tx Transaction. * @param commit Commit flag. */ - private void checkNormalTxFinish(Transaction tx, boolean commit) { + protected void checkNormalTxFinish(Transaction tx, boolean commit) { IgniteInternalTx tx0 = ((TransactionProxyImpl)tx).tx(); if (commit) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractByteArrayValuesSelfTest.java index cabe41f23d2cd..a3f348188acf3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractByteArrayValuesSelfTest.java @@ -27,18 +27,12 @@ * Base class for various tests for byte array values. */ public abstract class GridCacheAbstractByteArrayValuesSelfTest extends GridCommonAbstractTest { - /** Regular cache name. */ - protected static final String CACHE_REGULAR = "cache"; - /** Key 1. */ protected static final Integer KEY_1 = 1; /** Key 2. */ protected static final Integer KEY_2 = 2; - /** Use special key for swap test, otherwise entry with readers is not evicted. */ - protected static final Integer SWAP_TEST_KEY = 3; - /** Shared IP finder. */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllSelfTest.java index cae58e1816e4d..053345d57ec53 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheClearAllSelfTest.java @@ -28,6 +28,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -66,6 +67,14 @@ public class GridCacheClearAllSelfTest extends GridCommonAbstractTest { /** Cache mode which will be passed to grid configuration. */ private String cacheName = CACHE_NAME; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java index 3f4efc293e045..6ca7136f81000 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheConfigurationConsistencySelfTest.java @@ -51,6 +51,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -516,6 +517,32 @@ public void testDifferentAtomicity() throws Exception { ); } + /** + * @throws Exception If failed. + */ + public void testDifferentTxAtomicity() throws Exception { + cacheMode = PARTITIONED; + + checkSecondGridStartFails( + new C1() { + /** {@inheritDoc} */ + @Override public Void apply(CacheConfiguration cfg) { + cfg.setNearConfiguration(null); + cfg.setAtomicityMode(TRANSACTIONAL); + return null; + } + }, + new C1() { + /** {@inheritDoc} */ + @Override public Void apply(CacheConfiguration cfg) { + cfg.setNearConfiguration(null); + cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + return null; + } + } + ); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java index 016f3a099afc5..9aea277aa967f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -115,12 +116,26 @@ public class GridCacheEntryMemorySizeSelfTest extends GridCommonAbstractTest { startGrids(2); } + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + super.afterTest(); + } + /** * @param nearEnabled {@code True} if near cache should be enabled. * @param mode Cache mode. * @return Created cache. */ - private IgniteCache testCache(boolean nearEnabled, CacheMode mode) { + private IgniteCache createCache(boolean nearEnabled, CacheMode mode) { CacheConfiguration cacheCfg = defaultCacheConfiguration(); cacheCfg.setCacheMode(mode); @@ -177,7 +192,9 @@ protected Marshaller createMarshaller() throws IgniteCheckedException { /** @throws Exception If failed. */ public void testLocal() throws Exception { - IgniteCache cache = testCache(false, LOCAL); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + IgniteCache cache = createCache(false, LOCAL); try { cache.put(1, new Value(new byte[1024])); @@ -200,7 +217,7 @@ public void testLocal() throws Exception { /** @throws Exception If failed. */ public void testReplicated() throws Exception { - IgniteCache cache = testCache(false, REPLICATED); + IgniteCache cache = createCache(false, REPLICATED); try { cache.put(1, new Value(new byte[1024])); @@ -223,7 +240,9 @@ public void testReplicated() throws Exception { /** @throws Exception If failed. */ public void testPartitionedNearEnabled() throws Exception { - IgniteCache cache = testCache(true, PARTITIONED); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + IgniteCache cache = createCache(true, PARTITIONED); try { int[] keys = new int[3]; @@ -275,7 +294,7 @@ public void testPartitionedNearEnabled() throws Exception { /** @throws Exception If failed. */ public void testPartitionedNearDisabled() throws Exception { - IgniteCache cache = testCache(false, PARTITIONED); + IgniteCache cache = createCache(false, PARTITIONED); try { int[] keys = new int[3]; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLocalTxStoreExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLocalTxStoreExceptionSelfTest.java index 71f1f7f9765e0..fccf5ecd9bf4e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLocalTxStoreExceptionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLocalTxStoreExceptionSelfTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -25,6 +26,13 @@ * */ public class GridCacheLocalTxStoreExceptionSelfTest extends IgniteTxStoreExceptionAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMixedPartitionExchangeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMixedPartitionExchangeSelfTest.java index f6f47517e4de2..320dacded69d9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMixedPartitionExchangeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMixedPartitionExchangeSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -53,6 +54,14 @@ public class GridCacheMixedPartitionExchangeSelfTest extends GridCommonAbstractT /** Flag indicating whether to include cache to the node configuration. */ private boolean cache; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccMultiThreadedUpdateSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccMultiThreadedUpdateSelfTest.java new file mode 100644 index 0000000000000..4d9fb111396be --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMvccMultiThreadedUpdateSelfTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.concurrent.Callable; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteTransactions; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; +import org.apache.ignite.transactions.Transaction; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * Multithreaded update test. + */ +public class GridCacheMvccMultiThreadedUpdateSelfTest extends GridCacheOffHeapMultiThreadedUpdateAbstractSelfTest { + /** */ + public static final int THREADS = 5; + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return 5 * 60_000; + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); + + super.beforeTestsStarted(); + } + + /** + * @throws Exception If failed. + */ + public void testTransformTx() throws Exception { + testTransformTx(keyForNode(0)); + + if (gridCount() > 1) + testTransformTx(keyForNode(1)); + } + + /** + * @param key Key. + * @throws Exception If failed. + */ + private void testTransformTx(final Integer key) throws Exception { + final IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); + + cache.put(key, 0); + + final int ITERATIONS_PER_THREAD = iterations(); + + GridTestUtils.runMultiThreaded(new Callable() { + @Override public Void call() throws Exception { + IgniteTransactions txs = ignite(0).transactions(); + + for (int i = 0; i < ITERATIONS_PER_THREAD && !failed; i++) { + if (i % 500 == 0) + log.info("Iteration " + i); + + try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.invoke(key, new IncProcessor()); + + tx.commit(); + } + } + + return null; + } + }, THREADS, "transform"); + + for (int i = 0; i < gridCount(); i++) { + Integer val = (Integer)grid(i).cache(DEFAULT_CACHE_NAME).get(key); + + assertEquals("Unexpected value for grid " + i, (Integer)(ITERATIONS_PER_THREAD * THREADS), val); + } + + if (failed) { + for (int g = 0; g < gridCount(); g++) + info("Value for cache [g=" + g + ", val=" + grid(g).cache(DEFAULT_CACHE_NAME).get(key) + ']'); + + assertFalse(failed); + } + } + + /** + * @throws Exception If failed. + */ + public void testPutTxPessimistic() throws Exception { + testPutTx(keyForNode(0)); + + if (gridCount() > 1) + testPutTx(keyForNode(1)); + } + + /** + * @param key Key. + * @throws Exception If failed. + */ + private void testPutTx(final Integer key) throws Exception { + final IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); + + cache.put(key, 0); + + final int ITERATIONS_PER_THREAD = iterations(); + + GridTestUtils.runMultiThreaded(new Callable() { + @Override public Void call() throws Exception { + for (int i = 0; i < ITERATIONS_PER_THREAD; i++) { + if (i % 500 == 0) + log.info("Iteration " + i); + + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + Integer val = cache.getAndPut(key, i); + + assertNotNull(val); + + tx.commit(); + } + } + + return null; + } + }, THREADS, "put"); + + for (int i = 0; i < gridCount(); i++) { + Integer val = (Integer)grid(i).cache(DEFAULT_CACHE_NAME).get(key); + + assertNotNull("Unexpected value for grid " + i, val); + } + } + + /** + * @throws Exception If failed. + */ + public void testPutxIfAbsentTxPessimistic() throws Exception { + testPutxIfAbsentTx(keyForNode(0)); + + if (gridCount() > 1) + testPutxIfAbsentTx(keyForNode(1)); + } + + /** + * @param key Key. + * @throws Exception If failed. + */ + private void testPutxIfAbsentTx(final Integer key) throws Exception { + final IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); + + cache.put(key, 0); + + final int ITERATIONS_PER_THREAD = iterations(); + + GridTestUtils.runMultiThreaded(new Callable() { + @Override public Void call() throws Exception { + for (int i = 0; i < ITERATIONS_PER_THREAD && !failed; i++) { + if (i % 500 == 0) + log.info("Iteration " + i); + + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + cache.putIfAbsent(key, 100); + + tx.commit(); + } + } + + return null; + } + }, THREADS, "putxIfAbsent"); + + for (int i = 0; i < gridCount(); i++) { + Integer val = (Integer)grid(i).cache(DEFAULT_CACHE_NAME).get(key); + + assertEquals("Unexpected value for grid " + i, (Integer)0, val); + } + + assertFalse(failed); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheObjectToStringSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheObjectToStringSelfTest.java index 33b7033df911e..bf87035e1fbae 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheObjectToStringSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheObjectToStringSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -56,6 +57,13 @@ public class GridCacheObjectToStringSelfTest extends GridCommonAbstractTest { /** Near enabled flag. */ private boolean nearEnabled; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheP2PUndeploySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheP2PUndeploySelfTest.java index 8de8e04d0b61c..65668747e8251 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheP2PUndeploySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheP2PUndeploySelfTest.java @@ -52,9 +52,6 @@ public class GridCacheP2PUndeploySelfTest extends GridCommonAbstractTest { /** Test p2p value. */ private static final String TEST_VALUE = "org.apache.ignite.tests.p2p.GridCacheDeploymentTestValue3"; - /** */ - private static final long OFFHEAP = 0;// 4 * 1024 * 1024; - /** */ private final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStopSelfTest.java index 7f82d75d02ca1..eed5fdb35f5a5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStopSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStopSelfTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteInternalFuture; @@ -42,6 +43,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; @@ -52,14 +54,11 @@ * Tests correct cache stopping. */ public class GridCacheStopSelfTest extends GridCommonAbstractTest { - /** */ - private static final String EXPECTED_MSG = "Cache has been closed or destroyed"; - /** */ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); /** */ - private boolean atomic; + private CacheAtomicityMode atomicityMode = TRANSACTIONAL; /** */ private boolean replicated; @@ -74,15 +73,14 @@ public class GridCacheStopSelfTest extends GridCommonAbstractTest { cfg.setDiscoverySpi(disc); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setCacheMode(replicated ? REPLICATED : PARTITIONED); + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAtomicityMode(atomicityMode); - if (!replicated) + if (replicated) + ccfg.setCacheMode(REPLICATED); + else ccfg.setBackups(1); - ccfg.setAtomicityMode(atomic ? ATOMIC : TRANSACTIONAL); - cfg.setCacheConfiguration(ccfg); return cfg; @@ -97,6 +95,8 @@ public class GridCacheStopSelfTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testStopExplicitTransactions() throws Exception { + atomicityMode = TRANSACTIONAL; + testStop(true); } @@ -104,6 +104,8 @@ public void testStopExplicitTransactions() throws Exception { * @throws Exception If failed. */ public void testStopImplicitTransactions() throws Exception { + atomicityMode = TRANSACTIONAL; + testStop(false); } @@ -111,6 +113,7 @@ public void testStopImplicitTransactions() throws Exception { * @throws Exception If failed. */ public void testStopExplicitTransactionsReplicated() throws Exception { + atomicityMode = TRANSACTIONAL; replicated = true; testStop(true); @@ -120,6 +123,45 @@ public void testStopExplicitTransactionsReplicated() throws Exception { * @throws Exception If failed. */ public void testStopImplicitTransactionsReplicated() throws Exception { + atomicityMode = TRANSACTIONAL; + replicated = true; + + testStop(false); + } + + /** + * @throws Exception If failed. + */ + public void testStopExplicitMvccTransactions() throws Exception { + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + testStop(true); + } + + /** + * @throws Exception If failed. + */ + public void testStopImplicitMvccTransactions() throws Exception { + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + testStop(false); + } + + /** + * @throws Exception If failed. + */ + public void testStopExplicitMvccTransactionsReplicated() throws Exception { + atomicityMode = TRANSACTIONAL_SNAPSHOT; + replicated = true; + + testStop(true); + } + + /** + * @throws Exception If failed. + */ + public void testStopImplicitMvccTransactionsReplicated() throws Exception { + atomicityMode = TRANSACTIONAL_SNAPSHOT; replicated = true; testStop(false); @@ -129,7 +171,7 @@ public void testStopImplicitTransactionsReplicated() throws Exception { * @throws Exception If failed. */ public void testStopAtomic() throws Exception { - atomic = true; + atomicityMode = ATOMIC; testStop(false); } @@ -218,6 +260,7 @@ public void testStopMultithreaded() throws Exception { * @param node Node. * @param cache Cache. */ + @SuppressWarnings("unchecked") private void cacheOperations(Ignite node, IgniteCache cache) { ThreadLocalRandom rnd = ThreadLocalRandom.current(); @@ -227,10 +270,12 @@ private void cacheOperations(Ignite node, IgniteCache cache) { cache.get(key); - try (Transaction tx = node.transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - cache.put(key, key); + if (cache.getConfiguration(CacheConfiguration.class).getAtomicityMode() != TRANSACTIONAL_SNAPSHOT) { + try (Transaction tx = node.transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + cache.put(key, key); - tx.commit(); + tx.commit(); + } } try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { @@ -260,7 +305,7 @@ private void testStop(final boolean startTx) throws Exception { CacheConfiguration ccfg = cache.getConfiguration(CacheConfiguration.class); - assertEquals(atomic ? ATOMIC : TRANSACTIONAL, ccfg.getAtomicityMode()); + assertEquals(atomicityMode, ccfg.getAtomicityMode()); assertEquals(replicated ? REPLICATED : PARTITIONED, ccfg.getCacheMode()); Collection> putFuts = new ArrayList<>(); @@ -272,7 +317,8 @@ private void testStop(final boolean startTx) throws Exception { @Override public Void call() throws Exception { try { if (startTx) { - TransactionConcurrency concurrency = key % 2 == 0 ? OPTIMISTIC : PESSIMISTIC; + TransactionConcurrency concurrency = + atomicityMode != TRANSACTIONAL_SNAPSHOT && (key % 2 == 0) ? OPTIMISTIC : PESSIMISTIC; try (Transaction tx = grid(0).transactions().txStart(concurrency, REPEATABLE_READ)) { cache.put(key, key); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStorePutxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStorePutxSelfTest.java deleted file mode 100644 index 2175abb43d39e..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStorePutxSelfTest.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.store.CacheStore; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.lang.IgniteBiInClosure; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.apache.ignite.transactions.Transaction; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheMode.PARTITIONED; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; - -/** - * Tests for reproduce problem with GG-6895: - * putx calls CacheStore.load() when null GridPredicate passed in to avoid IDE warnings - */ -public class GridCacheStorePutxSelfTest extends GridCommonAbstractTest { - /** */ - private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** */ - private static AtomicInteger loads; - - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setCacheMode(PARTITIONED); - ccfg.setAtomicityMode(TRANSACTIONAL); - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setCacheStoreFactory(singletonFactory(new TestStore())); - ccfg.setReadThrough(true); - ccfg.setWriteThrough(true); - ccfg.setLoadPreviousValue(true); - - cfg.setCacheConfiguration(ccfg); - - TcpDiscoverySpi disco = new TcpDiscoverySpi(); - - disco.setIpFinder(IP_FINDER); - - cfg.setDiscoverySpi(disco); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - loads = new AtomicInteger(); - - startGrid(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - stopGrid(); - } - - /** - * @throws Exception If failed. - */ - public void testPutShouldNotTriggerLoad() throws Exception { - jcache().put(1, 1); - jcache().put(2, 2); - - assertEquals(0, loads.get()); - } - - /** - * @throws Exception If failed. - */ - public void testPutShouldNotTriggerLoadWithTx() throws Exception { - IgniteCache cache = jcache(); - - try (Transaction tx = grid().transactions().txStart()) { - cache.put(1, 1); - cache.put(2, 2); - - tx.commit(); - } - - assertEquals(0, loads.get()); - } - - /** */ - private static class TestStore implements CacheStore { - /** {@inheritDoc} */ - @Nullable @Override public Integer load(Integer key) { - loads.incrementAndGet(); - - return null; - } - - /** {@inheritDoc} */ - @Override public void loadCache(IgniteBiInClosure clo, @Nullable Object... args) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public Map loadAll(Iterable keys) { - return Collections.emptyMap(); - } - - /** {@inheritDoc} */ - @Override public void write(javax.cache.Cache.Entry entry) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void writeAll(Collection> entries) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void delete(Object key) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void deleteAll(Collection keys) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void sessionEnd(boolean commit) { - // No-op. - } - } -} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerEvictionSelfTest.java index 66ef47c8be5d2..e981e2b6c186e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerEvictionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerEvictionSelfTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -49,6 +50,14 @@ public class GridCacheTtlManagerEvictionSelfTest extends GridCommonAbstractTest /** Cache mode. */ private volatile CacheMode cacheMode; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerSelfTest.java index 52f19b78ca65a..c48913e1a2a91 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManagerSelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -46,6 +47,13 @@ public class GridCacheTtlManagerSelfTest extends GridCommonAbstractTest { /** Test cache mode. */ protected CacheMode cacheMode; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTxPartitionedLocalStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTxPartitionedLocalStoreSelfTest.java index 272665b7555d5..9000c95f51e11 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTxPartitionedLocalStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTxPartitionedLocalStoreSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -27,6 +28,13 @@ * */ public class GridCacheTxPartitionedLocalStoreSelfTest extends GridCacheAbstractLocalStoreSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode getAtomicMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java index b4e564faa8ebd..5e06933da518c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java @@ -67,6 +67,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.jetbrains.annotations.Nullable; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -103,6 +104,13 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb /** */ private static AtomicBoolean serialized = new AtomicBoolean(false); + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerEagerTtlDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerEagerTtlDisabledTest.java index a1bf26b1632dd..90e28618eac33 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerEagerTtlDisabledTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerEagerTtlDisabledTest.java @@ -19,11 +19,19 @@ import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Tests expire events when {@link CacheConfiguration#isEagerTtl()} is disabled. */ public class IgniteCacheEntryListenerEagerTtlDisabledTest extends IgniteCacheEntryListenerTxTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EXPIRATION); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected boolean eagerTtl() { return false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerTxLocalTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerTxLocalTest.java index 5da10c626ca24..0212ddd6d983c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerTxLocalTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerTxLocalTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,13 @@ * */ public class IgniteCacheEntryListenerTxLocalTest extends IgniteCacheEntryListenerAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java index 4efe51303cbca..34f97a10c8c31 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java @@ -37,6 +37,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; @@ -99,7 +100,7 @@ public class IgniteCacheEntryProcessorCallTest extends GridCommonAbstractTest { /** * @throws Exception If failed. */ - public void testEntryProcessorCall() throws Exception { + public void testEntryProcessorCallOnAtomicCache() throws Exception { { CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); ccfg.setBackups(1); @@ -117,7 +118,12 @@ public void testEntryProcessorCall() throws Exception { checkEntryProcessorCallCount(ccfg, 1); } + } + /** + * @throws Exception If failed. + */ + public void testEntryProcessorCallOnTxCache() throws Exception { { CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); ccfg.setBackups(1); @@ -137,6 +143,29 @@ public void testEntryProcessorCall() throws Exception { } } + /** + * @throws Exception If failed. + */ + public void testEntryProcessorCallOnMvccCache() throws Exception { + { + CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + ccfg.setBackups(1); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + checkEntryProcessorCallCount(ccfg, 2); + } + + { + CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + ccfg.setBackups(0); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + checkEntryProcessorCallCount(ccfg, 1); + } + } + /** * @param ccfg Cache configuration. * @param expCallCnt Expected entry processor calls count. @@ -163,18 +192,22 @@ private void checkEntryProcessorCallCount(CacheConfiguration if (ccfg.getAtomicityMode() == TRANSACTIONAL) { checkEntryProcessCall(key++, clientCache1, OPTIMISTIC, REPEATABLE_READ, expCallCnt + 1); - checkEntryProcessCall(key++, clientCache1, PESSIMISTIC, REPEATABLE_READ, expCallCnt + 1); checkEntryProcessCall(key++, clientCache1, OPTIMISTIC, SERIALIZABLE, expCallCnt + 1); + checkEntryProcessCall(key++, clientCache1, PESSIMISTIC, REPEATABLE_READ, expCallCnt + 1); } + else if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + checkEntryProcessCall(key++, clientCache1, PESSIMISTIC, REPEATABLE_READ, expCallCnt); for (int i = 100; i < 110; i++) { checkEntryProcessCall(key++, srvCache, null, null, expCallCnt); if (ccfg.getAtomicityMode() == TRANSACTIONAL) { - checkEntryProcessCall(key++, srvCache, OPTIMISTIC, REPEATABLE_READ, expCallCnt + 1); - checkEntryProcessCall(key++, srvCache, PESSIMISTIC, REPEATABLE_READ, expCallCnt + 1); - checkEntryProcessCall(key++, srvCache, OPTIMISTIC, SERIALIZABLE, expCallCnt + 1); + checkEntryProcessCall(key++, clientCache1, OPTIMISTIC, REPEATABLE_READ, expCallCnt + 1); + checkEntryProcessCall(key++, clientCache1, OPTIMISTIC, SERIALIZABLE, expCallCnt + 1); + checkEntryProcessCall(key++, clientCache1, PESSIMISTIC, REPEATABLE_READ, expCallCnt + 1); } + else if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + checkEntryProcessCall(key++, clientCache1, PESSIMISTIC, REPEATABLE_READ, expCallCnt); } for (int i = 0; i < NODES; i++) @@ -182,7 +215,6 @@ private void checkEntryProcessorCallCount(CacheConfiguration } /** - * * @param key Key. * @param cache Cache. * @param concurrency Transaction concurrency. @@ -205,6 +237,9 @@ private void checkEntryProcessCall(Integer key, ", concurrency=" + concurrency + ", isolation=" + isolation + "]"); + int expCallCntOnGet = cache.getConfiguration(CacheConfiguration.class).getAtomicityMode() == TRANSACTIONAL_SNAPSHOT ? + 1 : expCallCnt; + Transaction tx; TestReturnValue retVal; @@ -237,7 +272,7 @@ private void checkEntryProcessCall(Integer key, if (tx != null) tx.commit(); - assertEquals(expCallCnt, callCnt.get()); + assertEquals(expCallCntOnGet, callCnt.get()); checkReturnValue(retVal, "0"); checkCacheValue(cache.getName(), key, new TestValue(0)); @@ -415,7 +450,7 @@ public Integer value() { if (o == null || getClass() != o.getClass()) return false; - TestValue testVal = (TestValue) o; + TestValue testVal = (TestValue)o; return val.equals(testVal.val); @@ -473,7 +508,7 @@ public Object argument() { if (o == null || getClass() != o.getClass()) return false; - TestReturnValue testVal = (TestReturnValue) o; + TestReturnValue testVal = (TestReturnValue)o; return val.equals(testVal.val); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java index d9a04280eff3b..28d65f77624da 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java @@ -49,6 +49,7 @@ import org.apache.ignite.transactions.TransactionConcurrency; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; @@ -70,10 +71,11 @@ public void testInvoke() throws Exception { invoke(cache, null); - if (atomicityMode() == TRANSACTIONAL) { - invoke(cache, PESSIMISTIC); + if (atomicityMode() != ATOMIC) { + invoke(cache, PESSIMISTIC); // Tx or Mvcc tx. - invoke(cache, OPTIMISTIC); + if (atomicityMode() == TRANSACTIONAL) + invoke(cache, OPTIMISTIC); } } @@ -235,10 +237,11 @@ public void testInvokeAll() throws Exception { invokeAll(cache, null); - if (atomicityMode() == TRANSACTIONAL) { + if (atomicityMode() != ATOMIC) { invokeAll(cache, PESSIMISTIC); - invokeAll(cache, OPTIMISTIC); + if (atomicityMode() == TRANSACTIONAL) + invokeAll(cache, OPTIMISTIC); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/context/IgniteCachePartitionedExecutionContextTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java similarity index 65% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/context/IgniteCachePartitionedExecutionContextTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java index 36d2c2b7f3262..7029605d8c4b5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/context/IgniteCachePartitionedExecutionContextTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java @@ -15,30 +15,43 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache.context; +package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; -import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; /** * */ -public class IgniteCachePartitionedExecutionContextTest extends IgniteCacheAbstractExecutionContextTest { +public class IgniteCacheMvccTxInvokeTest extends IgniteCacheInvokeAbstractTest { + /** {@inheritDoc} */ + @Override protected int gridCount() { + return 3; + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { - return CacheMode.PARTITIONED; + return PARTITIONED; } /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { - return ATOMIC; + return TRANSACTIONAL_SNAPSHOT; } /** {@inheritDoc} */ @Override protected NearCacheConfiguration nearConfiguration() { return null; } + + /** {@inheritDoc} */ + @Override public void testInvokeAllAppliedOnceOnBinaryTypeRegistration() { + fail("https://issues.apache.org/jira/browse/IGNITE-10472"); + + super.testInvokeAllAppliedOnceOnBinaryTypeRegistration(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxNearEnabledInvokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxNearEnabledInvokeTest.java new file mode 100644 index 0000000000000..78efd460d16ec --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxNearEnabledInvokeTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.configuration.NearCacheConfiguration; + +/** + * + */ +public class IgniteCacheMvccTxNearEnabledInvokeTest extends IgniteCacheMvccTxInvokeTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + super.beforeTestsStarted(); + } + + /** {@inheritDoc} */ + @Override protected NearCacheConfiguration nearConfiguration() { + return new NearCacheConfiguration(); + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java index 37fe5fa830ad5..63d3ae8d74056 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -45,6 +46,8 @@ public class IgniteCacheNearLockValueSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + startGrid(1); startGrid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalInvokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalInvokeTest.java index 55cc431dde553..107ad7dae88e0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalInvokeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalInvokeTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxLocalInvokeTest extends IgniteCacheInvokeAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllLargeBatchSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllLargeBatchSelfTest.java index 56a438107c40f..76649bf8a8f70 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllLargeBatchSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllLargeBatchSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -52,6 +53,13 @@ public class IgnitePutAllLargeBatchSelfTest extends GridCommonAbstractTest { /** Backups. */ private int backups = 1; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllUpdateNonPreloadedPartitionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllUpdateNonPreloadedPartitionSelfTest.java index 2503e219455e7..0f9e62baf5746 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllUpdateNonPreloadedPartitionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePutAllUpdateNonPreloadedPartitionSelfTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -41,6 +42,13 @@ public class IgnitePutAllUpdateNonPreloadedPartitionSelfTest extends GridCommonA /** Backups. */ private int backups = 1; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java index 680381a21d799..a8a7f33883ba9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java @@ -44,13 +44,14 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionTimeoutException; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; -import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; /** * Test checks that grid transaction configuration doesn't influence system caches. @@ -109,6 +110,9 @@ public CacheAtomicityMode atomicityMode() { * @throws Exception If failed. */ public void testUserTxTimeout() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + final Ignite ignite = grid(0); final IgniteCache cache = ignite.getOrCreateCache(CACHE_NAME); @@ -214,7 +218,7 @@ protected void checkExplicitTxTimeout(final IgniteCache cache, f * @throws Exception If failed. */ protected void checkStartTxSuccess(final IgniteInternalCache cache) throws Exception { - try (final GridNearTxLocal tx = CU.txStartInternal(cache.context(), cache, PESSIMISTIC, READ_COMMITTED)) { + try (final GridNearTxLocal tx = CU.txStartInternal(cache.context(), cache, PESSIMISTIC, REPEATABLE_READ)) { assert tx != null; sleepForTxFailure(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java index ac294b05067fb..6033ace80eae2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java @@ -41,6 +41,7 @@ import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.apache.ignite.spi.indexing.IndexingSpi; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionHeuristicException; @@ -77,8 +78,6 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac cfg.setIndexingSpi(new TestIndexingSpi()); - cfg.getTransactionConfiguration().setTxSerializableEnabled(true); - return cfg; } @@ -89,7 +88,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac ccfg.setCacheStoreFactory(null); ccfg.setReadThrough(false); ccfg.setWriteThrough(false); - ccfg.setLoadPreviousValue(true); + ccfg.setLoadPreviousValue(false); ccfg.setIndexedTypes(Integer.class, Integer.class); @@ -98,6 +97,9 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10377"); + super.beforeTestsStarted(); lastKey = 0; @@ -191,6 +193,9 @@ public void testRemoveNear() throws Exception { * @throws Exception If failed. */ public void testRemovePrimary() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); + checkRemove(false, keyForNode(grid(0).localNode(), PRIMARY)); checkRemove(true, keyForNode(grid(0).localNode(), PRIMARY)); @@ -311,6 +316,10 @@ public void testPutMultipleKeysTx() throws Exception { */ private void checkPutTx(boolean putBefore, TransactionConcurrency concurrency, TransactionIsolation isolation, final Integer... keys) throws Exception { + if (MvccFeatureChecker.forcedMvcc() && + !MvccFeatureChecker.isSupported(concurrency, isolation)) + return; + assertTrue(keys.length > 0); info("Test transaction [concurrency=" + concurrency + ", isolation=" + isolation + ']'); @@ -477,6 +486,9 @@ private void checkPut(boolean putBefore, final Integer key) throws Exception { * @throws Exception If failed. */ private void checkTransform(boolean putBefore, final Integer key) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); + if (putBefore) { TestIndexingSpi.forceFail(false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java index 863ab38ad74ea..30016daaf82ae 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException; import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -94,6 +95,8 @@ public abstract class IgniteTxStoreExceptionAbstractSelfTest extends GridCacheAb /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + store = new TestStore(); super.beforeTestsStarted(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/CacheKeepBinaryWithInterceptorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/CacheKeepBinaryWithInterceptorTest.java index 6cef6d20c2790..667b305e610f4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/CacheKeepBinaryWithInterceptorTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/CacheKeepBinaryWithInterceptorTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.binary; +import javax.cache.Cache; import org.apache.ignite.IgniteCache; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.CacheAtomicityMode; @@ -30,10 +31,10 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; -import javax.cache.Cache; - -import static org.apache.ignite.cache.CacheAtomicityMode.*; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*; +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** * @@ -55,16 +56,18 @@ public class CacheKeepBinaryWithInterceptorTest extends GridCommonAbstractTest { } /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); + @Override protected void afterTest() throws Exception { + stopAllGrids(); - startGrid(0); + super.afterTest(); } /** * @throws Exception If failed. */ public void testKeepBinaryWithInterceptor() throws Exception { + startGrid(0); + keepBinaryWithInterceptor(cacheConfiguration(ATOMIC, false)); keepBinaryWithInterceptor(cacheConfiguration(TRANSACTIONAL, false)); @@ -80,6 +83,23 @@ public void testKeepBinaryWithInterceptor() throws Exception { keepBinaryWithInterceptorPrimitives(cacheConfiguration(TRANSACTIONAL, true)); } + /** + * @throws Exception If failed. + */ + public void testKeepBinaryWithInterceptorOnMvccCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9323"); + + startGrid(0); + + keepBinaryWithInterceptor(cacheConfiguration(TRANSACTIONAL_SNAPSHOT, false)); + keepBinaryWithInterceptorPrimitives(cacheConfiguration(TRANSACTIONAL_SNAPSHOT, true)); + + startGridsMultiThreaded(1, 3); + + keepBinaryWithInterceptor(cacheConfiguration(TRANSACTIONAL_SNAPSHOT, false)); + keepBinaryWithInterceptorPrimitives(cacheConfiguration(TRANSACTIONAL_SNAPSHOT, true)); + } + /** * @param ccfg Cache configuration. */ @@ -230,7 +250,8 @@ static class TestInterceptor1 implements CacheInterceptor entry, BinaryObject newVal) { + @Nullable @Override public BinaryObject onBeforePut(Cache.Entry entry, + BinaryObject newVal) { System.out.println("Before put [e=" + entry + ", newVal=" + newVal + ']'); onBeforePut++; @@ -255,7 +276,8 @@ static class TestInterceptor1 implements CacheInterceptor onBeforeRemove(Cache.Entry entry) { + @Nullable @Override public IgniteBiTuple onBeforeRemove( + Cache.Entry entry) { assertEquals(1, (int)entry.getKey().field("key")); assertEquals(10, (int)entry.getValue().field("val")); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/DataStreamProcessorPersistenceBinarySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/DataStreamProcessorPersistenceBinarySelfTest.java new file mode 100644 index 0000000000000..84fb997375273 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/DataStreamProcessorPersistenceBinarySelfTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.binary.datastreaming; + +/** + * + */ +public class DataStreamProcessorPersistenceBinarySelfTest extends DataStreamProcessorBinarySelfTest { + /** {@inheritDoc} */ + @Override public boolean persistenceEnabled() { + return true; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/GridDataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/GridDataStreamerImplSelfTest.java index c629871333952..1496792d427cc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/GridDataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/datastreaming/GridDataStreamerImplSelfTest.java @@ -258,7 +258,7 @@ public void testAddBinaryCreatedWithBuilder() throws Exception { BinaryObjectBuilder obj = g0.binary().builder("NoExistedClass"); obj.setField("id", i); - obj.setField("name", String.valueOf("name = " + i)); + obj.setField("name", "name = " + i); dataLdr.addData(i, obj.build()); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTxNearUpdateTopologyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTxNearUpdateTopologyChangeTest.java index df02bb963ddf5..f00f56b78cd47 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTxNearUpdateTopologyChangeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheTxNearUpdateTopologyChangeTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.internal.processors.cache.CacheNearUpdateTopologyChangeAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -26,6 +27,13 @@ * */ public class CacheTxNearUpdateTopologyChangeTest extends CacheNearUpdateTopologyChangeAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java index a5126302685fe..a212e47b0ae8b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractDistributedByteArrayValuesSelfTest.java @@ -17,11 +17,9 @@ package org.apache.ignite.internal.processors.cache.distributed; -import java.util.Arrays; -import java.util.Collections; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheAbstractByteArrayValuesSelfTest; @@ -40,17 +38,31 @@ */ public abstract class GridCacheAbstractDistributedByteArrayValuesSelfTest extends GridCacheAbstractByteArrayValuesSelfTest { - /** Grids. */ - protected static Ignite[] ignites; + /** */ + private static final String CACHE = "cache"; + + /** */ + private static final String MVCC_CACHE = "mvccCache"; /** Regular caches. */ private static IgniteCache[] caches; + /** Regular caches. */ + private static IgniteCache[] mvccCaches; + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); - c.setCacheConfiguration(cacheConfiguration()); + CacheConfiguration mvccCfg = cacheConfiguration(MVCC_CACHE) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(null); // TODO IGNITE-7187: remove near cache disabling. + + + CacheConfiguration ccfg = cacheConfiguration(CACHE); + + c.setCacheConfiguration(ccfg, mvccCfg); c.setPeerClassLoadingEnabled(peerClassLoading()); @@ -70,12 +82,13 @@ protected int gridCount() { } /** + * @param name Cache name. * @return Cache configuration. */ - protected CacheConfiguration cacheConfiguration() { + protected CacheConfiguration cacheConfiguration(String name) { CacheConfiguration cfg = cacheConfiguration0(); - cfg.setName(CACHE_REGULAR); + cfg.setName(name); return cfg; } @@ -88,26 +101,31 @@ protected CacheConfiguration cacheConfiguration() { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + int gridCnt = gridCount(); assert gridCnt > 0; - ignites = new Ignite[gridCnt]; - caches = new IgniteCache[gridCnt]; + mvccCaches = new IgniteCache[gridCnt]; - for (int i = 0; i < gridCnt; i++) { - ignites[i] = startGrid(i); + startGridsMultiThreaded(gridCnt); - caches[i] = ignites[i].cache(CACHE_REGULAR); + for (int i = 0; i < gridCnt; i++) { + caches[i] = grid(i).cache(CACHE); + mvccCaches[i] = grid(i).cache(MVCC_CACHE); } } /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { caches = null; + mvccCaches = null; - ignites = null; + stopAllGrids(); + + super.afterTestsStopped(); } /** @@ -146,6 +164,25 @@ public void testOptimisticMixed() throws Exception { testTransactionMixed0(caches, OPTIMISTIC, KEY_1, wrap(1), KEY_2, 1); } + /** + * Check whether cache with byte array entry works correctly in PESSIMISTIC transaction. + * + * @throws Exception If failed. + */ + public void testPessimisticMvcc() throws Exception { + testTransaction0(mvccCaches, PESSIMISTIC, KEY_1, wrap(1)); + } + + /** + * Check whether cache with byte array entry works correctly in PESSIMISTIC transaction. + * + * @throws Exception If failed. + */ + public void testPessimisticMvccMixed() throws Exception { + testTransactionMixed0(mvccCaches, PESSIMISTIC, KEY_1, wrap(1), KEY_2, 1); + } + + /** * Test transaction behavior. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java index 0ee626213b3c4..824cdf5a9b941 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheAbstractPartitionedByteArrayValuesSelfTest.java @@ -18,10 +18,7 @@ package org.apache.ignite.internal.processors.cache.distributed; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; -import org.apache.ignite.configuration.TransactionConfiguration; -import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -32,31 +29,9 @@ */ public abstract class GridCacheAbstractPartitionedByteArrayValuesSelfTest extends GridCacheAbstractDistributedByteArrayValuesSelfTest { - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - if (nearConfiguration() != null) - MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); - - super.beforeTestsStarted(); - } - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - TransactionConfiguration tCfg = new TransactionConfiguration(); - - tCfg.setTxSerializableEnabled(true); - - cfg.setTransactionConfiguration(tCfg); - - return cfg; - } - /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration0() { - CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + CacheConfiguration cfg = new CacheConfiguration(); cfg.setCacheMode(PARTITIONED); cfg.setAtomicityMode(TRANSACTIONAL); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java index 5241c37f680c8..cc92ec13418a7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java @@ -75,7 +75,8 @@ public class IgniteCacheConnectionRecoveryTest extends GridCommonAbstractTest { cfg.setCacheConfiguration( cacheConfiguration("cache1", TRANSACTIONAL), - cacheConfiguration("cache2", ATOMIC)); + //cacheConfiguration("cache2", TRANSACTIONAL_SNAPSHOT), //TODO IGNITE-10474: add Mvcc cache after fix. + cacheConfiguration("cache3", ATOMIC)); return cfg; } @@ -94,6 +95,7 @@ public class IgniteCacheConnectionRecoveryTest extends GridCommonAbstractTest { /** * @throws Exception If failed. */ + @SuppressWarnings("unchecked") public void testConnectionRecovery() throws Exception { final Map data = new TreeMap<>(); @@ -115,16 +117,17 @@ public void testConnectionRecovery() throws Exception { Thread.currentThread().setName("test-thread-" + idx0 + "-" + node.name()); - IgniteCache cache1 = node.cache("cache1"); - IgniteCache cache2 = node.cache("cache2"); + IgniteCache[] caches = { + node.cache("cache1"), +// node.cache("cache2"), //TODO IGNITE-10474: add Mvcc cache after fix. + node.cache("cache3")}; int iter = 0; while (U.currentTimeMillis() < stopTime) { try { - cache1.putAllAsync(data).get(15, SECONDS); - - cache2.putAllAsync(data).get(15, SECONDS); + for (IgniteCache cache : caches) + cache.putAllAsync(data).get(15, SECONDS); CyclicBarrier b = barrierRef.get(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryIdleConnectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryIdleConnectionTest.java index 0f4aaa7a85c17..7689bc67776a0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryIdleConnectionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryIdleConnectionTest.java @@ -37,6 +37,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -88,6 +89,13 @@ public void testCacheOperationsIdleConnectionCloseTx() throws Exception { cacheOperationsIdleConnectionClose(TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testCacheOperationsIdleConnectionCloseMvccTx() throws Exception { + cacheOperationsIdleConnectionClose(TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSystemTransactionsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSystemTransactionsSelfTest.java index eb19873c816f5..1d63587350ef2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSystemTransactionsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSystemTransactionsSelfTest.java @@ -19,9 +19,8 @@ import java.util.Map; import org.apache.ignite.IgniteCache; -import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteKernal; -import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; @@ -30,6 +29,8 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.MvccFeatureChecker; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -41,19 +42,22 @@ /** * Tests that system transactions do not interact with user transactions. */ -public class IgniteCacheSystemTransactionsSelfTest extends GridCacheAbstractSelfTest { +public class IgniteCacheSystemTransactionsSelfTest extends GridCommonAbstractTest { + /** */ + private static final int NODES_CNT = 4; + /** {@inheritDoc} */ - @Override protected int gridCount() { - return 4; + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGridsMultiThreaded(NODES_CNT); } /** {@inheritDoc} */ - @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { - CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName); + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); - ccfg.setAtomicityMode(TRANSACTIONAL); - - return ccfg; + super.afterTestsStopped(); } /** {@inheritDoc} */ @@ -67,10 +71,19 @@ public class IgniteCacheSystemTransactionsSelfTest extends GridCacheAbstractSelf } } + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setCacheConfiguration(defaultCacheConfiguration().setAtomicityMode(TRANSACTIONAL)); + } + /** * @throws Exception If failed. */ public void testSystemTxInsideUserTx() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10473"); + IgniteKernal ignite = (IgniteKernal)grid(0); IgniteCache jcache = ignite.cache(DEFAULT_CACHE_NAME); @@ -100,8 +113,8 @@ public void testSystemTxInsideUserTx() throws Exception { checkTransactionsCommitted(); - checkEntries(DEFAULT_CACHE_NAME, "1", "11", "2", "22", "3", null); - checkEntries(CU.UTILITY_CACHE_NAME, "1", null, "2", "2", "3", "3"); + checkEntries(DEFAULT_CACHE_NAME, "1", "11", "2", "22", "3", null); + checkEntries(CU.UTILITY_CACHE_NAME, "1", null, "2", "2", "3", "3"); } /** @@ -112,7 +125,9 @@ public void testGridNearTxLocalDuplicateAsyncCommit() throws Exception { IgniteInternalCache utilityCache = ignite.context().cache().utilityCache(); - try (GridNearTxLocal itx = utilityCache.txStartEx(OPTIMISTIC, SERIALIZABLE)) { + try (GridNearTxLocal itx = MvccFeatureChecker.forcedMvcc() ? + utilityCache.txStartEx(PESSIMISTIC, REPEATABLE_READ) : + utilityCache.txStartEx(OPTIMISTIC, SERIALIZABLE)) { utilityCache.put("1", "1"); itx.commitNearTxLocalAsync(); @@ -124,7 +139,7 @@ public void testGridNearTxLocalDuplicateAsyncCommit() throws Exception { * @throws Exception If failed. */ private void checkTransactionsCommitted() throws Exception { - for (int i = 0; i < gridCount(); i++) { + for (int i = 0; i < NODES_CNT; i++) { IgniteKernal kernal = (IgniteKernal)grid(i); IgniteTxManager tm = kernal.context().cache().context().tm(); @@ -149,7 +164,7 @@ private void checkTransactionsCommitted() throws Exception { * @throws Exception If failed. */ private void checkEntries(String cacheName, Object... vals) throws Exception { - for (int g = 0; g < gridCount(); g++) { + for (int g = 0; g < NODES_CNT; g++) { IgniteKernal kernal = (IgniteKernal)grid(g); GridCacheAdapter cache = kernal.context().cache().internalCache(cacheName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java index 66453b80ca75a..60c083775557a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java @@ -17,15 +17,15 @@ package org.apache.ignite.internal.processors.cache.distributed; -import javax.cache.Cache; -import javax.cache.configuration.Factory; -import javax.cache.integration.CacheLoaderException; -import javax.cache.integration.CacheWriterException; import java.util.Collection; import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; +import javax.cache.Cache; +import javax.cache.configuration.Factory; +import javax.cache.integration.CacheLoaderException; +import javax.cache.integration.CacheWriterException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.store.CacheStore; @@ -37,6 +37,7 @@ import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.resources.CacheStoreSessionResource; import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.Nullable; @@ -90,6 +91,8 @@ private CacheConfiguration cacheConfiguration(String cacheName, Factory[] cachesAtomic; @@ -43,10 +44,16 @@ public abstract class GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest ex CacheConfiguration atomicCacheCfg = cacheConfiguration0(); - atomicCacheCfg.setName(CACHE_ATOMIC); + atomicCacheCfg.setName(ATOMIC_CACHE); atomicCacheCfg.setAtomicityMode(ATOMIC); - c.setCacheConfiguration(cacheConfiguration(), atomicCacheCfg); + int size = c.getCacheConfiguration().length; + + CacheConfiguration[] configs = Arrays.copyOf(c.getCacheConfiguration(), size + 1); + + configs[size] = atomicCacheCfg; + + c.setCacheConfiguration(configs); c.setPeerClassLoadingEnabled(peerClassLoading()); @@ -68,7 +75,7 @@ public abstract class GridCacheAbstractPartitionedOnlyByteArrayValuesSelfTest ex cachesAtomic = new IgniteCache[gridCnt]; for (int i = 0; i < gridCount(); i++) - cachesAtomic[i] = ignites[i].cache(CACHE_ATOMIC); + cachesAtomic[i] = grid(i).cache(ATOMIC_CACHE); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheGlobalLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheGlobalLoadTest.java index 6024030ab6f15..63b244915e7fc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheGlobalLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheGlobalLoadTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -69,6 +70,13 @@ public class GridCacheGlobalLoadTest extends IgniteCacheAbstractTest { return new NearCacheConfiguration(); } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAbstractNearPartitionedByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAbstractNearPartitionedByteArrayValuesSelfTest.java index c6c849ac9bd42..188c364c7978d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAbstractNearPartitionedByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAbstractNearPartitionedByteArrayValuesSelfTest.java @@ -29,4 +29,18 @@ public abstract class GridCacheAbstractNearPartitionedByteArrayValuesSelfTest ex @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); } + + /** {@inheritDoc} */ + @Override public void testPessimisticMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + super.testPessimisticMvcc(); + } + + /** {@inheritDoc} */ + @Override public void testPessimisticMvccMixed() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + super.testPessimisticMvccMixed(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheGetStoreErrorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheGetStoreErrorSelfTest.java index cb537ee67dfcf..013f885747b36 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheGetStoreErrorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheGetStoreErrorSelfTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -55,6 +56,14 @@ public class GridCacheGetStoreErrorSelfTest extends GridCommonAbstractTest { /** Cache mode for test. */ private CacheMode cacheMode; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -116,7 +125,7 @@ private void checkGetError(boolean nearEnabled, CacheMode cacheMode) throws Exce this.nearEnabled = nearEnabled; this.cacheMode = cacheMode; - startGrids(3); + startGridsMultiThreaded(3); try { GridTestUtils.assertThrows(log, new Callable() { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxExceptionSelfTest.java index d6e3804728cb4..277e829013756 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxExceptionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxExceptionSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.IgniteTxExceptionAbstractSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -26,6 +27,13 @@ * Tests near cache. */ public class GridCacheNearTxExceptionSelfTest extends IgniteTxExceptionAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return PARTITIONED; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedStorePutSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedStorePutSelfTest.java index 96d4603fb3cc0..3f72756965394 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedStorePutSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedStorePutSelfTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.distributed.near; import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.Cache; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.store.CacheStoreAdapter; import org.apache.ignite.configuration.CacheConfiguration; @@ -28,7 +29,9 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -45,13 +48,14 @@ public class GridCachePartitionedStorePutSelfTest extends GridCommonAbstractTest private static final AtomicInteger CNT = new AtomicInteger(0); /** */ - private IgniteCache cache1; + private static AtomicInteger loads; - /** */ - private IgniteCache cache2; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); - /** */ - private IgniteCache cache3; + super.beforeTestsStarted(); + } /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -95,9 +99,9 @@ private CacheConfiguration cacheConfiguration() { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - cache1 = startGrid(1).cache(DEFAULT_CACHE_NAME); - cache2 = startGrid(2).cache(DEFAULT_CACHE_NAME); - cache3 = startGrid(3).cache(DEFAULT_CACHE_NAME); + loads = new AtomicInteger(); + + startGridsMultiThreaded(3); } /** {@inheritDoc} */ @@ -105,21 +109,35 @@ private CacheConfiguration cacheConfiguration() { stopAllGrids(); } - /** - * @throws Exception If failed. - */ - public void testPutx() throws Throwable { - info("Putting to the first node."); + /** */ + public void testPutShouldNotTriggerLoad() { + checkPut(0); - cache1.put(0, 1); + assertEquals(0, loads.get()); - info("Putting to the second node."); + checkPut(1); - cache2.put(0, 2); + assertEquals(0, loads.get()); - info("Putting to the third node."); + checkPut(2); + + assertEquals(0, loads.get()); + } + + /** */ + public void checkPut(int idx) { + IgniteCache cache = grid(idx).cache(DEFAULT_CACHE_NAME); + + cache.put(0, 1); + + try (Transaction tx = grid(idx).transactions().txStart()) { + cache.put(1, 1); + cache.put(2, 2); + + tx.commit(); + } - cache3.put(0, 3); + assertEquals(0, loads.get()); } /** @@ -128,13 +146,13 @@ public void testPutx() throws Throwable { private static class TestStore extends CacheStoreAdapter { /** {@inheritDoc} */ @Override public Object load(Object key) { - assert false; + loads.incrementAndGet(); return null; } /** {@inheritDoc} */ - @Override public void write(javax.cache.Cache.Entry e) { + @Override public void write(Cache.Entry e) { // No-op } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheAbstractReplicatedByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheAbstractReplicatedByteArrayValuesSelfTest.java index 9fd2f29c15623..b4e58fcc4ec6b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheAbstractReplicatedByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheAbstractReplicatedByteArrayValuesSelfTest.java @@ -41,7 +41,7 @@ public abstract class GridCacheAbstractReplicatedByteArrayValuesSelfTest extends /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration0() { - CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + CacheConfiguration cfg = new CacheConfiguration(); cfg.setCacheMode(REPLICATED); cfg.setAtomicityMode(TRANSACTIONAL); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java index e6c4771a90aad..63f910fbca860 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalByteArrayValuesSelfTest.java @@ -56,7 +56,6 @@ public class GridCacheLocalByteArrayValuesSelfTest extends GridCacheAbstractByte CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - ccfg.setName(CACHE_REGULAR); ccfg.setAtomicityMode(TRANSACTIONAL); ccfg.setCacheMode(LOCAL); ccfg.setWriteSynchronizationMode(FULL_SYNC); @@ -72,14 +71,14 @@ public class GridCacheLocalByteArrayValuesSelfTest extends GridCacheAbstractByte ignite = startGrid(1); - cache = ignite.cache(CACHE_REGULAR); + cache = ignite.cache(DEFAULT_CACHE_NAME); } /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { cache = null; - ignite = null; + stopAllGrids(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxExceptionSelfTest.java index 63a900dd1141d..ae1e2d0e62d10 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxExceptionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxExceptionSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.IgniteTxExceptionAbstractSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -26,6 +27,13 @@ * Tests local cache. */ public class GridCacheLocalTxExceptionSelfTest extends IgniteTxExceptionAbstractSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java index 435ccc05eeb31..6e4a9e5c77526 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java @@ -45,6 +45,7 @@ import org.jetbrains.annotations.NotNull; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; @@ -73,13 +74,13 @@ public class CacheEntryProcessorExternalizableFailedTest extends GridCommonAbstr public static final int ITERATION_CNT = 1; /** */ - public static final int KEYS = 10; + public static final int KEY = 10; /** */ private boolean client; /** */ - private boolean failOnWrite = false; + private boolean failOnWrite; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -139,6 +140,157 @@ public void testOptimisticFullSync() throws Exception { doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); } + /** + * @throws Exception If failed. + */ + public void testOptimistic() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testOptimisticWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testOptimisticFullSyncWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testOptimisticOnePhaseCommit() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testOptimisticOnePhaseCommitWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testOptimisticOnePhaseCommitFullSync() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testOptimisticOnePhaseCommitFullSyncWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + } + /** * @throws Exception If failed. */ @@ -165,7 +317,7 @@ public void testPessimisticOnePhaseCommit() throws Exception { */ public void testPessimisticOnePhaseCommitWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); @@ -208,7 +360,7 @@ public void testPessimisticOnePhaseCommitFullSync() throws Exception { */ public void testPessimisticOnePhaseCommitFullSyncWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); @@ -251,7 +403,7 @@ public void testPessimistic() throws Exception { */ public void testPessimisticWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); @@ -292,63 +444,83 @@ public void testPessimisticFullSync() throws Exception { /** * @throws Exception If failed. */ - public void testOptimisticOnePhaseCommit() throws Exception { - CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1); + public void testPessimisticFullSyncWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2) + .setNearConfiguration(new NearCacheConfiguration<>()); - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); failOnWrite = true; - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); } /** * @throws Exception If failed. */ - public void testOptimisticOnePhaseCommitFullSync() throws Exception { - CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1); - - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + public void testMvccPessimisticOnePhaseCommit() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); failOnWrite = true; - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + /** + * @throws Exception If failed. + */ + public void testMvccPessimisticOnePhaseCommitWithNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + + failOnWrite = true; + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); } /** * @throws Exception If failed. */ - public void testOptimisticOnePhaseCommitFullSyncWithNearCache() throws Exception { - CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1) - .setNearConfiguration(new NearCacheConfiguration()); + public void testMvccPessimisticOnePhaseCommitFullSync() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); - - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); failOnWrite = true; - doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } - doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); + /** + * @throws Exception If failed. + */ + public void testMvccPessimisticOnePhaseCommitFullSyncWithNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + + failOnWrite = true; doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); } @@ -356,20 +528,30 @@ public void testOptimisticOnePhaseCommitFullSyncWithNearCache() throws Exception /** * @throws Exception If failed. */ - public void testOptimistic() throws Exception { - CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2); + public void testMvccPessimistic() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); - - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); failOnWrite = true; - doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } - doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); + /** + * @throws Exception If failed. + */ + public void testMvccPessimisticWithNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + + failOnWrite = true; doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); } @@ -377,20 +559,30 @@ public void testOptimistic() throws Exception { /** * @throws Exception If failed. */ - public void testOptimisticFullSyncWithNearCache() throws Exception { - CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2); + public void testMvccPessimisticFullSync() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); - - doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); - doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); failOnWrite = true; - doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } - doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); + /** + * @throws Exception If failed. + */ + public void testMvccPessimisticFullSyncWithNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + + failOnWrite = true; doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); } @@ -399,6 +591,7 @@ public void testOptimisticFullSyncWithNearCache() throws Exception { * @param ccfg Cache configuration. * @throws Exception If failed. */ + @SuppressWarnings("unchecked") private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency txConcurrency, TransactionIsolation txIsolation) throws Exception { IgniteEx cln = grid(getServerNodeCount()); @@ -412,28 +605,19 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx else clnCache = cln.cache(ccfg.getName()); - putKeys(clnCache, EXPECTED_VALUE); + clnCache.put(KEY, EXPECTED_VALUE); try { // Explicit tx. for (int i = 0; i < ITERATION_CNT; i++) { - try (final Transaction tx = cln.transactions().txStart(txConcurrency, txIsolation)) { - putKeys(clnCache, WRONG_VALUE); - - clnCache.invoke(KEYS, createEntryProcessor()); - - GridTestUtils.assertThrowsWithCause(new Callable() { - @Override public Object call() throws Exception { - tx.commit(); - - return null; - } - }, UnsupportedOperationException.class); - } + if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + checkExplicitMvccInvoke(cln, clnCache, txConcurrency, txIsolation); + else + checkExplicitTxInvoke(cln, clnCache, txConcurrency, txIsolation); assertNull(cln.transactions().tx()); - checkKeys(clnCache, EXPECTED_VALUE); + assertEquals(EXPECTED_VALUE, clnCache.get(KEY)); } // From affinity node. @@ -443,32 +627,24 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx // Explicit tx. for (int i = 0; i < ITERATION_CNT; i++) { - try (final Transaction tx = grid.transactions().txStart(txConcurrency, txIsolation)) { - putKeys(cache, WRONG_VALUE); - - cache.invoke(KEYS, createEntryProcessor()); - - GridTestUtils.assertThrowsWithCause(new Callable() { - @Override public Object call() throws Exception { - tx.commit(); - - return null; - } - }, UnsupportedOperationException.class); - } + if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + checkExplicitMvccInvoke(cln, clnCache, txConcurrency, txIsolation); + else + checkExplicitTxInvoke(cln, clnCache, txConcurrency, txIsolation); assertNull(cln.transactions().tx()); - checkKeys(cache, EXPECTED_VALUE); + assertEquals(EXPECTED_VALUE, cache.get(KEY)); } final IgniteCache clnCache0 = clnCache; // Implicit tx. for (int i = 0; i < ITERATION_CNT; i++) { + //noinspection ThrowableNotThrown GridTestUtils.assertThrowsWithCause(new Callable() { @Override public Object call() throws Exception { - clnCache0.invoke(KEYS, createEntryProcessor()); + clnCache0.invoke(KEY, createEntryProcessor()); return null; } @@ -477,7 +653,7 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx assertNull(cln.transactions().tx()); } - checkKeys(clnCache, EXPECTED_VALUE); + assertEquals(EXPECTED_VALUE, clnCache.get(KEY)); } catch (Exception e) { e.printStackTrace(); @@ -488,33 +664,61 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx } /** - * @return Entry processor. + * @param node Ignite node. + * @param cache Node cache. + * @param txConcurrency Transaction concurrency. + * @param txIsolation TransactionIsolation. */ - @NotNull private EntryProcessor createEntryProcessor() { - return failOnWrite ? new ExternalizableFailedWriteEntryProcessor() : - new ExternalizableFailedReadEntryProcessor(); + @SuppressWarnings({"unchecked", "ThrowableNotThrown"}) + private void checkExplicitTxInvoke(Ignite node, IgniteCache cache, TransactionConcurrency txConcurrency, + TransactionIsolation txIsolation) { + try (final Transaction tx = node.transactions().txStart(txConcurrency, txIsolation)) { + cache.put(KEY, WRONG_VALUE); + + cache.invoke(KEY, createEntryProcessor()); + + GridTestUtils.assertThrowsWithCause(new Callable() { + @Override public Object call() throws Exception { + tx.commit(); + + return null; + } + }, UnsupportedOperationException.class); + } } - /** - * @param cache Cache. - * @param val Value. - */ - private void putKeys(IgniteCache cache, int val) { - cache.put(KEYS, val); + @SuppressWarnings({"unchecked", "ThrowableNotThrown"}) + private void checkExplicitMvccInvoke(Ignite node, IgniteCache cache, TransactionConcurrency txConcurrency, + TransactionIsolation txIsolation) { + try (final Transaction tx = node.transactions().txStart(txConcurrency, txIsolation)) { + cache.put(KEY, WRONG_VALUE); + + GridTestUtils.assertThrowsWithCause(new Callable() { + @Override public Object call() throws Exception { + cache.invoke(KEY, createEntryProcessor()); + + fail("Should never happened."); + + tx.commit(); + + return null; + } + }, UnsupportedOperationException.class); + } } /** - * @param cache Cache. - * @param expVal Expected value. + * @return Entry processor. */ - private void checkKeys(IgniteCache cache, int expVal) { - assertEquals(expVal, cache.get(KEYS)); + private @NotNull EntryProcessor createEntryProcessor() { + return failOnWrite ? new ExternalizableFailedWriteEntryProcessor() : + new ExternalizableFailedReadEntryProcessor(); } /** * @return Cache configuration. */ - private CacheConfiguration cacheConfiguration(CacheWriteSynchronizationMode wrMode, int backup) { + private CacheConfiguration cacheConfiguration(CacheWriteSynchronizationMode wrMode, int backup) { return new CacheConfiguration("test-cache-" + wrMode + "-" + backup) .setAtomicityMode(TRANSACTIONAL) .setWriteSynchronizationMode(FULL_SYNC) @@ -525,7 +729,7 @@ private CacheConfiguration cacheConfiguration(CacheWriteSynchronizationMode wrMo * */ private static class ExternalizableFailedWriteEntryProcessor implements EntryProcessor, - Externalizable{ + Externalizable { /** */ public ExternalizableFailedWriteEntryProcessor() { // No-op. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java index 693668556ffb9..ad03bf5a91199 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java @@ -36,12 +36,14 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; @@ -70,7 +72,7 @@ public class CacheEntryProcessorNonSerializableTest extends GridCommonAbstractTe public static final int ITERATION_CNT = 1; /** */ - public static final int KEYS = 10; + private static final int KEY = 10; /** */ private boolean client; @@ -125,7 +127,7 @@ public void testPessimisticOnePhaseCommit() throws Exception { */ public void testPessimisticOnePhaseCommitWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); @@ -152,7 +154,7 @@ public void testPessimisticOnePhaseCommitFullSync() throws Exception { */ public void testPessimisticOnePhaseCommitFullSyncWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); @@ -179,7 +181,7 @@ public void testPessimistic() throws Exception { */ public void testPessimisticWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); @@ -201,6 +203,101 @@ public void testPessimisticFullSync() throws Exception { doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); } + /** + * @throws Exception If failed. + */ + public void testPessimisticFullSyncWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + + doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); + } + + /** + */ + public void testMvccPessimisticOnePhaseCommit() { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + */ + public void testMvccPessimisticOnePhaseCommitWithNearCache() { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + */ + public void testMvccPessimisticOnePhaseCommitFullSync() { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + */ + public void testMvccPessimisticOnePhaseCommitFullSyncWithNearCache() { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + */ + public void testMvccPessimistic() { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + */ + public void testMvccPessimisticWithNearCache() { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + */ + public void testMvccPessimisticFullSync() { + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testMvccPessimisticFullSyncWithNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, PESSIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); + + doTestInvokeTest(ccfg, PESSIMISTIC, SERIALIZABLE); + } + /** * @throws Exception If failed. */ @@ -214,6 +311,20 @@ public void testOptimisticOnePhaseCommit() throws Exception { doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); } + /** + * @throws Exception If failed. + */ + public void testOptimisticOnePhaseCommitWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + } + /** * @throws Exception If failed. */ @@ -232,7 +343,7 @@ public void testOptimisticOnePhaseCommitFullSync() throws Exception { */ public void testOptimisticOnePhaseCommitFullSyncWithNearCache() throws Exception { CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1) - .setNearConfiguration(new NearCacheConfiguration()); + .setNearConfiguration(new NearCacheConfiguration<>()); doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); @@ -254,6 +365,20 @@ public void testOptimistic() throws Exception { doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); } + /** + * @throws Exception If failed. + */ + public void testOptimisticWithNearCache() throws Exception { + CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2) + .setNearConfiguration(new NearCacheConfiguration<>()); + + doTestInvokeTest(ccfg, OPTIMISTIC, READ_COMMITTED); + + doTestInvokeTest(ccfg, OPTIMISTIC, REPEATABLE_READ); + + doTestInvokeTest(ccfg, OPTIMISTIC, SERIALIZABLE); + } + /** * @throws Exception If failed. */ @@ -282,10 +407,13 @@ public void testOptimisticFullSyncWithNearCache() throws Exception { /** * @param ccfg Cache configuration. - * @throws Exception If failed. */ + @SuppressWarnings({"unchecked", "ThrowableNotThrown"}) private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency txConcurrency, - TransactionIsolation txIsolation) throws Exception { + TransactionIsolation txIsolation) { + if (ccfg.getNearConfiguration() != null) + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + IgniteEx cln = grid(getServerNodeCount()); grid(0).createCache(ccfg); @@ -297,26 +425,17 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx else clnCache = cln.cache(ccfg.getName()); - putKeys(clnCache, EXPECTED_VALUE); + clnCache.put(KEY, EXPECTED_VALUE); try { // Explicit tx. for (int i = 0; i < ITERATION_CNT; i++) { - try (final Transaction tx = cln.transactions().txStart(txConcurrency, txIsolation)) { - putKeys(clnCache, WRONG_VALUE); - - clnCache.invoke(KEYS, new NonSerialazibleEntryProcessor()); - - GridTestUtils.assertThrowsWithCause(new Callable() { - @Override public Object call() throws Exception { - tx.commit(); + if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + checkMvccInvoke(cln, clnCache, txConcurrency, txIsolation); + else + checkTxInvoke(cln, clnCache, txConcurrency, txIsolation); - return null; - } - }, NotSerializableException.class); - } - - checkKeys(clnCache, EXPECTED_VALUE); + assertEquals(EXPECTED_VALUE, clnCache.get(KEY)); } // From affinity node. @@ -326,21 +445,12 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx // Explicit tx. for (int i = 0; i < ITERATION_CNT; i++) { - try (final Transaction tx = grid.transactions().txStart(txConcurrency, txIsolation)) { - putKeys(cache, WRONG_VALUE); - - cache.invoke(KEYS, new NonSerialazibleEntryProcessor()); + if (ccfg.getAtomicityMode() == TRANSACTIONAL_SNAPSHOT) + checkMvccInvoke(grid, cache, txConcurrency, txIsolation); + else + checkTxInvoke(grid, cache, txConcurrency, txIsolation); - GridTestUtils.assertThrowsWithCause(new Callable() { - @Override public Object call() throws Exception { - tx.commit(); - - return null; - } - }, NotSerializableException.class); - } - - checkKeys(cache, EXPECTED_VALUE); + assertEquals(EXPECTED_VALUE, cache.get(KEY)); } final IgniteCache clnCache0 = clnCache; @@ -348,15 +458,15 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx // Implicit tx. for (int i = 0; i < ITERATION_CNT; i++) { GridTestUtils.assertThrowsWithCause(new Callable() { - @Override public Object call() throws Exception { - clnCache0.invoke(KEYS, new NonSerialazibleEntryProcessor()); + @Override public Object call() { + clnCache0.invoke(KEY, new NonSerialazibleEntryProcessor()); return null; } }, NotSerializableException.class); } - checkKeys(clnCache, EXPECTED_VALUE); + assertEquals(EXPECTED_VALUE, clnCache.get(KEY)); } finally { grid(0).destroyCache(ccfg.getName()); @@ -364,25 +474,59 @@ private void doTestInvokeTest(CacheConfiguration ccfg, TransactionConcurrency tx } /** - * @param cache Cache. - * @param val Value. + * @param node Grid node. + * @param cache Node cache. + * @param txConcurrency Transaction concurrency. + * @param txIsolation Transaction isolation. */ - private void putKeys(IgniteCache cache, int val) { - cache.put(KEYS, val); + @SuppressWarnings({"unchecked", "ThrowableNotThrown"}) + private void checkTxInvoke(Ignite node, IgniteCache cache, TransactionConcurrency txConcurrency, + TransactionIsolation txIsolation) { + try (final Transaction tx = node.transactions().txStart(txConcurrency, txIsolation)) { + cache.put(KEY, WRONG_VALUE); + + cache.invoke(KEY, new NonSerialazibleEntryProcessor()); + + GridTestUtils.assertThrowsWithCause(new Callable() { + @Override public Object call() { + tx.commit(); + + return null; + } + }, NotSerializableException.class); + } } /** - * @param cache Cache. - * @param expVal Expected value. + * @param node Grid node. + * @param cache Node cache. + * @param txConcurrency Transaction concurrency. + * @param txIsolation Transaction isolation. */ - private void checkKeys(IgniteCache cache, int expVal) { - assertEquals(expVal, cache.get(KEYS)); + @SuppressWarnings({"unchecked", "ThrowableNotThrown"}) + private void checkMvccInvoke(Ignite node, IgniteCache cache, TransactionConcurrency txConcurrency, + TransactionIsolation txIsolation) { + try (final Transaction tx = node.transactions().txStart(txConcurrency, txIsolation)) { + cache.put(KEY, WRONG_VALUE); + + GridTestUtils.assertThrowsWithCause(new Callable() { + @Override public Object call() { + cache.invoke(KEY, new NonSerialazibleEntryProcessor()); + + fail("Should never happened."); + + tx.commit(); + + return null; + } + }, NotSerializableException.class); + } } /** * @return Cache configuration. */ - private CacheConfiguration cacheConfiguration(CacheWriteSynchronizationMode wrMode, int backup) { + private CacheConfiguration cacheConfiguration(CacheWriteSynchronizationMode wrMode, int backup) { return new CacheConfiguration("test-cache-" + wrMode + "-" + backup) .setAtomicityMode(TRANSACTIONAL) .setWriteSynchronizationMode(FULL_SYNC) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java index 13c5e41c27cb3..cf07f20457582 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java @@ -38,6 +38,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -98,6 +99,9 @@ public class TxRollbackOnTopologyChangeTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9322"); //Won't start nodes if the only test mutes. + super.beforeTest(); startGridsMultiThreaded(TOTAL_CNT); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java index ca8be09a18bc2..03c9d1984c503 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java @@ -64,6 +64,6 @@ public class DataStreamProcessorMvccSelfTest extends DataStreamProcessorSelfTest /** {@inheritDoc} */ @Override public void testLocal() { - // Do not check local caches with MVCC enabled. + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java index e7f43a495d706..55a18199314ca 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorSelfTest.java @@ -67,12 +67,12 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.stream.StreamReceiver; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -131,7 +131,6 @@ public boolean persistenceEnabled() { TcpDiscoverySpi spi = new TcpDiscoverySpi(); - spi.setForceServerMode(true); spi.setIpFinder(ipFinder); cfg.setDiscoverySpi(spi); @@ -193,6 +192,8 @@ protected boolean customKeepBinary() { * @throws Exception If failed. */ public void testPartitioned() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + mode = PARTITIONED; checkDataStreamer(); @@ -221,6 +222,8 @@ public void testReplicated() throws Exception { * @throws Exception If failed. */ public void testLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = LOCAL; try { @@ -360,9 +363,7 @@ private void checkIsolatedDataStreamer() throws Exception { try { useCache = true; - Ignite g1 = startGrid(0); - startGrid(1); - startGrid(2); + Ignite g1 = startGridsMultiThreaded(3); afterGridStarted(); @@ -499,20 +500,15 @@ public void testPartitionedMultiThreaded() throws Exception { protected void checkLoaderMultithreaded(int nodesCntNoCache, int nodesCntCache) throws Exception { try { - // Start all required nodes. - int idx = 1; - useCache = true; - for (int i = 0; i < nodesCntCache; i++) - startGrid(idx++); + startGridsMultiThreaded(nodesCntCache); useCache = false; - for (int i = 0; i < nodesCntNoCache; i++) - startGrid(idx++); + startGridsMultiThreaded(nodesCntCache, nodesCntNoCache); - Ignite g1 = grid(idx - 1); + Ignite g1 = grid(nodesCntCache + nodesCntNoCache - 1); afterGridStarted(); @@ -771,8 +767,7 @@ private static IgniteClosure removeClosure(@Nullable final T exp) { * @throws Exception If failed. */ public void testFlush() throws Exception { - // Local caches are not allowed with MVCC enabled. - mode = getCacheAtomicityMode() != TRANSACTIONAL_SNAPSHOT ? LOCAL : PARTITIONED; + mode = PARTITIONED; useCache = true; @@ -826,8 +821,7 @@ public Void call() throws Exception { * @throws Exception If failed. */ public void testTryFlush() throws Exception { - // Local caches are not allowed with MVCC enabled. - mode = getCacheAtomicityMode() != TRANSACTIONAL_SNAPSHOT ? LOCAL : PARTITIONED; + mode = PARTITIONED; useCache = true; @@ -864,8 +858,9 @@ public void testTryFlush() throws Exception { * @throws Exception If failed. */ public void testFlushTimeout() throws Exception { - // Local caches are not allowed with MVCC enabled. - mode = getCacheAtomicityMode() != TRANSACTIONAL_SNAPSHOT ? LOCAL : PARTITIONED; + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + mode = PARTITIONED; useCache = true; @@ -918,6 +913,8 @@ public void testFlushTimeout() throws Exception { * @throws Exception If failed. */ public void testUpdateStore() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + storeMap = new ConcurrentHashMap<>(); try { @@ -925,10 +922,7 @@ public void testUpdateStore() throws Exception { useCache = true; - Ignite ignite = startGrid(1); - - startGrid(2); - startGrid(3); + Ignite ignite = startGridsMultiThreaded(3); afterGridStarted(); @@ -993,10 +987,7 @@ public void testCustomUserUpdater() throws Exception { useCache = true; try { - Ignite ignite = startGrid(1); - - startGrid(2); - startGrid(3); + Ignite ignite = startGridsMultiThreaded(3); afterGridStarted(); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index 40bea5a5cb3d7..1956d4b3e5523 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -42,6 +42,7 @@ import org.apache.ignite.internal.IgniteSlowClientDetectionSelfTest; import org.apache.ignite.internal.MarshallerContextLockingSelfTest; import org.apache.ignite.internal.TransactionsMXBeanImplTest; +import org.apache.ignite.internal.managers.IgniteDiagnosticMessagesMultipleConnectionsTest; import org.apache.ignite.internal.managers.IgniteDiagnosticMessagesTest; import org.apache.ignite.internal.processors.affinity.GridAffinityProcessorMemoryLeakTest; import org.apache.ignite.internal.processors.affinity.GridAffinityProcessorRendezvousSelfTest; @@ -197,6 +198,7 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex suite.addTestSuite(IgniteMarshallerCacheClientRequestsMappingOnMissTest.class); suite.addTestSuite(IgniteDiagnosticMessagesTest.class); + suite.addTestSuite(IgniteDiagnosticMessagesMultipleConnectionsTest.class); suite.addTestSuite(IgniteRejectConnectOnNodeStopTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java index 5cfe534d4c3b6..4d47808dc811b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheTestSuite.java @@ -17,15 +17,16 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import java.util.HashSet; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.GridCacheAffinityRoutingSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheEntryMemorySizeSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheMvccSelfTest; import org.apache.ignite.internal.processors.cache.binary.CacheKeepBinaryWithInterceptorTest; -import org.apache.ignite.internal.processors.cache.expiry.IgniteCacheAtomicLocalExpiryPolicyTest; import org.apache.ignite.internal.processors.cache.binary.GridBinaryCacheEntryMemorySizeSelfTest; import org.apache.ignite.internal.processors.cache.binary.datastreaming.DataStreamProcessorBinarySelfTest; +import org.apache.ignite.internal.processors.cache.binary.datastreaming.DataStreamProcessorPersistenceBinarySelfTest; import org.apache.ignite.internal.processors.cache.binary.datastreaming.GridDataStreamerImplSelfTest; import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAffinityRoutingBinarySelfTest; import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultiNodeSelfTest; @@ -34,8 +35,8 @@ import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryMultithreadedSelfTest; import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheBinariesNearPartitionedByteArrayValuesSelfTest; import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheBinariesPartitionedOnlyByteArrayValuesSelfTest; -import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorPersistenceSelfTest; -import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorSelfTest; +import org.apache.ignite.internal.processors.cache.expiry.IgniteCacheAtomicLocalExpiryPolicyTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Cache suite with binary marshaller. @@ -43,16 +44,19 @@ public class IgniteBinaryCacheTestSuite extends TestSuite { /** * @return Suite. - * @throws Exception In case of error. */ - public static TestSuite suite() throws Exception { - TestSuite suite = new TestSuite("Binary Cache Test Suite"); + public static TestSuite suite() { + return suite(new HashSet<>()); + } - HashSet ignoredTests = new HashSet<>(); + /** + * @param ignoredTests Tests to ignore. + * @return Test suite. + */ + public static TestSuite suite(Collection ignoredTests) { + TestSuite suite = new TestSuite("Binary Cache Test Suite"); // Tests below have a special version for Binary Marshaller - ignoredTests.add(DataStreamProcessorSelfTest.class); - ignoredTests.add(DataStreamProcessorPersistenceSelfTest.class); ignoredTests.add(GridCacheAffinityRoutingSelfTest.class); ignoredTests.add(IgniteCacheAtomicLocalExpiryPolicyTest.class); ignoredTests.add(GridCacheEntryMemorySizeSelfTest.class); @@ -62,23 +66,21 @@ public static TestSuite suite() throws Exception { suite.addTest(IgniteCacheTestSuite.suite(ignoredTests)); - // TODO GG-11148 - // suite.addTestSuite(GridCacheMemoryModeBinarySelfTest.class); - - suite.addTestSuite(GridCacheBinariesPartitionedOnlyByteArrayValuesSelfTest.class); - suite.addTestSuite(GridCacheBinariesNearPartitionedByteArrayValuesSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheBinariesPartitionedOnlyByteArrayValuesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheBinariesNearPartitionedByteArrayValuesSelfTest.class, ignoredTests); - suite.addTestSuite(GridDataStreamerImplSelfTest.class); - suite.addTestSuite(DataStreamProcessorBinarySelfTest.class); - suite.addTestSuite(GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultiNodeSelfTest.class); - suite.addTestSuite(GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultithreadedSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridDataStreamerImplSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorBinarySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorPersistenceBinarySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultithreadedSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheAtomicPartitionedOnlyBinaryMultiNodeSelfTest.class); - suite.addTestSuite(GridCacheAtomicPartitionedOnlyBinaryMultithreadedSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicPartitionedOnlyBinaryMultiNodeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicPartitionedOnlyBinaryMultithreadedSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheAffinityRoutingBinarySelfTest.class); - suite.addTestSuite(GridBinaryCacheEntryMemorySizeSelfTest.class); - suite.addTestSuite(CacheKeepBinaryWithInterceptorTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheAffinityRoutingBinarySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridBinaryCacheEntryMemorySizeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheKeepBinaryWithInterceptorTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite1.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite1.java new file mode 100755 index 0000000000000..cd56ff5eb9a48 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite1.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import java.util.Set; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest; +import org.apache.ignite.cache.IgniteWarmupClosureSelfTest; +import org.apache.ignite.cache.store.CacheStoreReadFromBackupTest; +import org.apache.ignite.cache.store.GridCacheBalancingStoreSelfTest; +import org.apache.ignite.cache.store.GridStoreLoadCacheTest; +import org.apache.ignite.cache.store.StoreResourceInjectionSelfTest; +import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest; +import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest; +import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest; +import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest; +import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalancePairedConnectionsTest; +import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest; +import org.apache.ignite.internal.managers.communication.IgniteCommunicationSslBalanceTest; +import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest; +import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest; +import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest; +import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteQueueTest; +import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest; +import org.apache.ignite.internal.processors.cache.CacheMvccTxFastFinishTest; +import org.apache.ignite.internal.processors.cache.CacheNamesSelfTest; +import org.apache.ignite.internal.processors.cache.CacheNamesWithSpecialCharactersTest; +import org.apache.ignite.internal.processors.cache.CacheTxFastFinishTest; +import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest; +import org.apache.ignite.internal.processors.cache.GridCacheAffinityApiSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheAffinityMapperSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheAffinityRoutingSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheAsyncOperationsLimitSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMapSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheConfigurationConsistencySelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheConfigurationValidationSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheLifecycleAwareSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheMissingCommitVersionSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheMvccManagerSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheMvccMultiThreadedUpdateSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheMvccPartitionedSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheMvccSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheOffHeapAtomicMultiThreadedUpdateSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheOffHeapMultiThreadedUpdateSelfTest; +import org.apache.ignite.internal.processors.cache.GridCachePartitionedLocalStoreSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheReplicatedLocalStoreSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheStopSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheSwapPreloadSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheTcpClientDiscoveryMultiThreadedTest; +import org.apache.ignite.internal.processors.cache.GridDataStorageConfigurationConsistencySelfTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalWithStoreInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicNearEnabledInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicStopBusySelfTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicWithStoreInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicLocalTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicReplicatedTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheEntryProcessorCallTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheManyAsyncOperationsTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheMvccTxInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheMvccTxNearEnabledInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheTxInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheTxNearEnabledInvokeTest; +import org.apache.ignite.internal.processors.cache.IgniteClientAffinityAssignmentSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteIncompleteCacheObjectSelfTest; +import org.apache.ignite.internal.processors.cache.binary.CacheKeepBinaryWithInterceptorTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAffinityRoutingBinarySelfTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultiNodeSelfTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultithreadedSelfTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryMultiNodeSelfTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheAtomicPartitionedOnlyBinaryMultithreadedSelfTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheBinariesNearPartitionedByteArrayValuesSelfTest; +import org.apache.ignite.internal.processors.cache.binary.distributed.dht.GridCacheBinariesPartitionedOnlyByteArrayValuesSelfTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCacheAtomicExecutionContextTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCacheContinuousExecutionContextTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCacheIsolatedExecutionContextTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCacheP2PDisableExecutionContextTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCachePrivateExecutionContextTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCacheReplicatedExecutionContextTest; +import org.apache.ignite.internal.processors.cache.context.IgniteCacheSharedExecutionContextTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicNearUpdateTopologyChangeTest; +import org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecovery10ConnectionsTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryPairedConnectionsTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecovery10ConnectionsTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecoveryTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheMessageRecoveryIdleConnectionTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheMessageWriteTimeoutTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicNearCacheSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionsStateValidatorSelfTest; +import org.apache.ignite.internal.processors.cache.expiry.IgniteCacheAtomicLocalExpiryPolicyTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheEntryProcessorExternalizableFailedTest; +import org.apache.ignite.internal.processors.cache.query.continuous.CacheEntryProcessorNonSerializableTest; + +/** + * Test suite. + */ +public class IgniteCacheMvccTestSuite1 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 1"); + + Set ignoredTests = new HashSet<>(); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(CacheKeepBinaryWithInterceptorTest.class); + ignoredTests.add(CacheEntryProcessorNonSerializableTest.class); + ignoredTests.add(CacheEntryProcessorExternalizableFailedTest.class); + ignoredTests.add(IgniteCacheEntryProcessorSequentialCallTest.class); + ignoredTests.add(IgniteCacheEntryProcessorCallTest.class); + ignoredTests.add(GridCacheConfigurationConsistencySelfTest.class); + ignoredTests.add(IgniteCacheMessageRecoveryIdleConnectionTest.class); + ignoredTests.add(IgniteCacheConnectionRecoveryTest.class); + ignoredTests.add(IgniteCacheConnectionRecovery10ConnectionsTest.class); + ignoredTests.add(CacheDeferredDeleteSanitySelfTest.class); + ignoredTests.add(CacheDeferredDeleteQueueTest.class); + ignoredTests.add(GridCacheStopSelfTest.class); + ignoredTests.add(GridCacheBinariesNearPartitionedByteArrayValuesSelfTest.class); + ignoredTests.add(GridCacheBinariesPartitionedOnlyByteArrayValuesSelfTest.class); + + // Atomic caches. + ignoredTests.add(IgniteCacheEntryListenerAtomicTest.class); + ignoredTests.add(IgniteCacheEntryListenerAtomicReplicatedTest.class); + ignoredTests.add(IgniteCacheEntryListenerAtomicLocalTest.class); + ignoredTests.add(IgniteCacheAtomicLocalExpiryPolicyTest.class); + ignoredTests.add(IgniteCacheAtomicInvokeTest.class); + ignoredTests.add(IgniteCacheAtomicNearEnabledInvokeTest.class); + ignoredTests.add(IgniteCacheAtomicWithStoreInvokeTest.class); + ignoredTests.add(IgniteCacheAtomicLocalInvokeTest.class); + ignoredTests.add(IgniteCacheAtomicLocalWithStoreInvokeTest.class); + ignoredTests.add(GridCachePartitionedLocalStoreSelfTest.class); + ignoredTests.add(GridCacheReplicatedLocalStoreSelfTest.class); + ignoredTests.add(CacheStoreReadFromBackupTest.class); + + ignoredTests.add(IgniteCacheAtomicExecutionContextTest.class); + ignoredTests.add(IgniteCacheReplicatedExecutionContextTest.class); + ignoredTests.add(IgniteCacheContinuousExecutionContextTest.class); + ignoredTests.add(IgniteCacheIsolatedExecutionContextTest.class); + ignoredTests.add(IgniteCacheP2PDisableExecutionContextTest.class); + ignoredTests.add(IgniteCachePrivateExecutionContextTest.class); + ignoredTests.add(IgniteCacheSharedExecutionContextTest.class); + + ignoredTests.add(IgniteCacheAtomicStopBusySelfTest.class); + ignoredTests.add(GridCacheAtomicNearCacheSelfTest.class); + ignoredTests.add(CacheAtomicNearUpdateTopologyChangeTest.class); + ignoredTests.add(GridCacheOffHeapAtomicMultiThreadedUpdateSelfTest.class); + ignoredTests.add(IgniteCacheAtomicMessageRecoveryTest.class); + ignoredTests.add(IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.class); + ignoredTests.add(IgniteCacheAtomicMessageRecovery10ConnectionsTest.class); + + ignoredTests.add(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearPartitionedAtomic.class); + ignoredTests.add(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearReplicatedAtomic.class); + ignoredTests.add(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientPartitionedAtomic.class); + ignoredTests.add(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientReplicatedAtomic.class); + + ignoredTests.add(GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultiNodeSelfTest.class); + ignoredTests.add(GridCacheAtomicPartitionedOnlyBinaryDataStreamerMultithreadedSelfTest.class); + ignoredTests.add(GridCacheAtomicPartitionedOnlyBinaryMultiNodeSelfTest.class); + ignoredTests.add(GridCacheAtomicPartitionedOnlyBinaryMultithreadedSelfTest.class); + + // Irrelevant tests. + ignoredTests.add(GridCacheMvccSelfTest.class); // This is about MvccCandidate, but not TxSnapshot. + ignoredTests.add(GridCacheMvccPartitionedSelfTest.class); // This is about MvccCandidate, but not TxSnapshot. + ignoredTests.add(GridCacheMvccManagerSelfTest.class); // This is about MvccCandidate, but not TxSnapshot. + ignoredTests.add(GridCacheSwapPreloadSelfTest.class); + ignoredTests.add(GridCacheMissingCommitVersionSelfTest.class); // Mvcc tx states resides in TxLog. + + // Other non-Tx test. + ignoredTests.add(GridCacheAffinityRoutingSelfTest.class); + ignoredTests.add(GridCacheAffinityRoutingBinarySelfTest.class); + ignoredTests.add(IgniteClientAffinityAssignmentSelfTest.class); + ignoredTests.add(GridCacheConcurrentMapSelfTest.class); + ignoredTests.add(CacheAffinityCallSelfTest.class); + ignoredTests.add(GridCacheAffinityMapperSelfTest.class); + ignoredTests.add(GridCacheAffinityApiSelfTest.class); + + ignoredTests.add(CacheNamesSelfTest.class); + ignoredTests.add(CacheNamesWithSpecialCharactersTest.class); + ignoredTests.add(GridCacheConfigurationValidationSelfTest.class); + + ignoredTests.add(GridDataStorageConfigurationConsistencySelfTest.class); + ignoredTests.add(DataStorageConfigurationValidationTest.class); + ignoredTests.add(JdbcTypesDefaultTransformerTest.class); + ignoredTests.add(GridCacheJdbcBlobStoreSelfTest.class); + ignoredTests.add(CacheJdbcPojoStoreTest.class); + ignoredTests.add(GridCacheBalancingStoreSelfTest.class); + ignoredTests.add(GridStoreLoadCacheTest.class); + + ignoredTests.add(IgniteWarmupClosureSelfTest.class); + ignoredTests.add(StoreResourceInjectionSelfTest.class); + ignoredTests.add(GridCacheAsyncOperationsLimitSelfTest.class); + ignoredTests.add(IgniteCacheManyAsyncOperationsTest.class); + ignoredTests.add(GridCacheLifecycleAwareSelfTest.class); + ignoredTests.add(IgniteCacheMessageWriteTimeoutTest.class); + ignoredTests.add(GridCachePartitionsStateValidatorSelfTest.class); + ignoredTests.add(IgniteVariousConnectionNumberTest.class); + ignoredTests.add(IgniteIncompleteCacheObjectSelfTest.class); + + ignoredTests.add(IgniteCommunicationBalanceTest.class); + ignoredTests.add(IgniteCommunicationBalancePairedConnectionsTest.class); + ignoredTests.add(IgniteCommunicationBalanceMultipleConnectionsTest.class); + ignoredTests.add(IgniteCommunicationSslBalanceTest.class); + ignoredTests.add(IgniteIoTestMessagesTest.class); + + ignoredTests.add(GridCacheTcpClientDiscoveryMultiThreadedTest.class); + + // Skip classes which Mvcc implementations are added in this method below. + ignoredTests.add(GridCacheOffHeapMultiThreadedUpdateSelfTest.class); // See GridCacheMvccMultiThreadedUpdateSelfTest. + ignoredTests.add(CacheTxFastFinishTest.class); // See CacheMvccTxFastFinishTest. + ignoredTests.add(IgniteCacheTxInvokeTest.class); // See IgniteCacheMvccTxInvokeTest. + ignoredTests.add(IgniteCacheTxNearEnabledInvokeTest.class); // See IgniteCacheMvccTxNearEnabledInvokeTest. + + suite.addTest(IgniteBinaryCacheTestSuite.suite(ignoredTests)); + + // Add Mvcc clones. + suite.addTestSuite(GridCacheMvccMultiThreadedUpdateSelfTest.class); + suite.addTestSuite(CacheMvccTxFastFinishTest.class); + suite.addTestSuite(IgniteCacheMvccTxInvokeTest.class); + suite.addTestSuite(IgniteCacheMvccTxNearEnabledInvokeTest.class); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTcpClientDiscoveryTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTcpClientDiscoveryTestSuite.java index e0efda43e4a8c..68c61e9edb497 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTcpClientDiscoveryTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTcpClientDiscoveryTestSuite.java @@ -17,9 +17,19 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.GridCacheTcpClientDiscoveryMultiThreadedTest; -import org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest; +import org.apache.ignite.testframework.GridTestUtils; + +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientPartitionedAtomic; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientPartitionedTransactional; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientReplicatedAtomic; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientReplicatedTransactional; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearPartitionedAtomic; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearPartitionedTransactional; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearReplicatedAtomic; +import static org.apache.ignite.internal.processors.cache.distributed.GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearReplicatedTransactional; /** * Tests a cache with TcpClientDiscovery SPI being enabled. @@ -27,20 +37,27 @@ public class IgniteCacheTcpClientDiscoveryTestSuite { /** * @return Suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Tests to ignore. + * @return Test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Cache + TcpClientDiscovery SPI test suite."); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearPartitionedAtomic.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearPartitionedTransactional.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearReplicatedAtomic.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseNearReplicatedTransactional.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientPartitionedAtomic.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientPartitionedTransactional.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientReplicatedAtomic.class)); - suite.addTest(new TestSuite(GridCacheClientModesTcpClientDiscoveryAbstractTest.CaseClientReplicatedTransactional.class)); - suite.addTest(new TestSuite(GridCacheTcpClientDiscoveryMultiThreadedTest.class)); + GridTestUtils.addTestIfNeeded(suite, CaseNearPartitionedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseNearPartitionedTransactional.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseNearReplicatedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseNearReplicatedTransactional.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseClientPartitionedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseClientPartitionedTransactional.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseClientReplicatedAtomic.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CaseClientReplicatedTransactional.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheTcpClientDiscoveryMultiThreadedTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java index eeec5e8b52482..63ae59dca9db9 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java @@ -17,7 +17,7 @@ package org.apache.ignite.testsuites; -import java.util.Set; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest; import org.apache.ignite.cache.IgniteWarmupClosureSelfTest; @@ -37,10 +37,6 @@ import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest; import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest; import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest; -import org.apache.ignite.internal.IgniteInternalCacheRemoveTest; -import org.apache.ignite.internal.managers.IgniteDiagnosticMessagesMultipleConnectionsTest; -import org.apache.ignite.internal.managers.IgniteDiagnosticMessagesTest; -import org.apache.ignite.internal.managers.communication.GridIoManagerSelfTest; import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest; import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalancePairedConnectionsTest; import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest; @@ -49,7 +45,6 @@ import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest; import org.apache.ignite.internal.processors.cache.BinaryMetadataRegistrationInsideEntryProcessorTest; import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest; -import org.apache.ignite.internal.processors.cache.CacheAtomicSingleMessageCountSelfTest; import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteQueueTest; import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest; import org.apache.ignite.internal.processors.cache.CacheFutureExceptionSelfTest; @@ -62,25 +57,16 @@ import org.apache.ignite.internal.processors.cache.GridCacheAffinityMapperSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheAffinityRoutingSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheAsyncOperationsLimitSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheAtomicUsersAffinityMapperSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheClearAllSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheClearLocallySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheColocatedTxStoreExceptionSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheConcurrentGetCacheOnClientTest; import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMapSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheConfigurationConsistencySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheConfigurationValidationSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheEntryMemorySizeSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheFullTextQueryMultithreadedSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheKeyCheckNearEnabledSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheKeyCheckSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheLeakTest; import org.apache.ignite.internal.processors.cache.GridCacheLifecycleAwareSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheLocalTxStoreExceptionSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheMissingCommitVersionSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheMixedPartitionExchangeSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheMultiUpdateLockSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheMvccFlagsTest; import org.apache.ignite.internal.processors.cache.GridCacheMvccManagerSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheMvccPartitionedSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheMvccSelfTest; @@ -91,17 +77,11 @@ import org.apache.ignite.internal.processors.cache.GridCachePartitionedLocalStoreSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheReplicatedLocalStoreSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheReplicatedTxStoreExceptionSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheReplicatedUsersAffinityMapperSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheReturnValueTransferSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheSlowTxWarnTest; import org.apache.ignite.internal.processors.cache.GridCacheStopSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheStorePutxSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheStoreValueBytesSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheSwapPreloadSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheTtlManagerLoadTest; import org.apache.ignite.internal.processors.cache.GridCacheTtlManagerSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheTxPartitionedLocalStoreSelfTest; -import org.apache.ignite.internal.processors.cache.GridCacheTxUsersAffinityMapperSelfTest; import org.apache.ignite.internal.processors.cache.GridDataStorageConfigurationConsistencySelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicInvokeTest; import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalInvokeTest; @@ -109,7 +89,6 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicNearEnabledInvokeTest; import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicStopBusySelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicWithStoreInvokeTest; -import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryEntryProcessorSelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicLocalTest; import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicReplicatedTest; import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest; @@ -120,27 +99,19 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheEntryProcessorCallTest; import org.apache.ignite.internal.processors.cache.IgniteCacheManyAsyncOperationsTest; import org.apache.ignite.internal.processors.cache.IgniteCacheNearLockValueSelfTest; -import org.apache.ignite.internal.processors.cache.IgniteCacheObjectPutSelfTest; -import org.apache.ignite.internal.processors.cache.IgniteCacheSerializationSelfTest; -import org.apache.ignite.internal.processors.cache.IgniteCacheStartStopLoadTest; import org.apache.ignite.internal.processors.cache.IgniteCacheTransactionalStopBusySelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheTxInvokeTest; import org.apache.ignite.internal.processors.cache.IgniteCacheTxLocalInvokeTest; import org.apache.ignite.internal.processors.cache.IgniteCacheTxNearEnabledInvokeTest; -import org.apache.ignite.internal.processors.cache.IgniteCachingProviderSelfTest; import org.apache.ignite.internal.processors.cache.IgniteClientAffinityAssignmentSelfTest; import org.apache.ignite.internal.processors.cache.IgniteIncompleteCacheObjectSelfTest; -import org.apache.ignite.internal.processors.cache.IgniteOnePhaseCommitNearSelfTest; import org.apache.ignite.internal.processors.cache.IgnitePutAllLargeBatchSelfTest; import org.apache.ignite.internal.processors.cache.IgnitePutAllUpdateNonPreloadedPartitionSelfTest; -import org.apache.ignite.internal.processors.cache.IgniteStaticCacheStartSelfTest; import org.apache.ignite.internal.processors.cache.IgniteTxConfigCacheSelfTest; -import org.apache.ignite.internal.processors.cache.InterceptorWithKeepBinaryCacheFullApiTest; import org.apache.ignite.internal.processors.cache.context.IgniteCacheAtomicExecutionContextTest; import org.apache.ignite.internal.processors.cache.context.IgniteCacheContinuousExecutionContextTest; import org.apache.ignite.internal.processors.cache.context.IgniteCacheIsolatedExecutionContextTest; import org.apache.ignite.internal.processors.cache.context.IgniteCacheP2PDisableExecutionContextTest; -import org.apache.ignite.internal.processors.cache.context.IgniteCachePartitionedExecutionContextTest; import org.apache.ignite.internal.processors.cache.context.IgniteCachePrivateExecutionContextTest; import org.apache.ignite.internal.processors.cache.context.IgniteCacheReplicatedExecutionContextTest; import org.apache.ignite.internal.processors.cache.context.IgniteCacheSharedExecutionContextTest; @@ -165,6 +136,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionsStateValidatorSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheGetStoreErrorSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxExceptionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedStorePutSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxExceptionSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxExceptionSelfTest; import org.apache.ignite.internal.processors.cache.query.continuous.CacheEntryProcessorExternalizableFailedTest; @@ -185,55 +157,52 @@ public class IgniteCacheTestSuite extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { return suite(null); } /** * @param ignoredTests Tests to ignore. * @return Test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite(Set ignoredTests) throws Exception { + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite"); - suite.addTestSuite(IgniteCacheEntryListenerAtomicTest.class); - suite.addTestSuite(IgniteCacheEntryListenerAtomicReplicatedTest.class); - suite.addTestSuite(IgniteCacheEntryListenerAtomicLocalTest.class); - suite.addTestSuite(IgniteCacheEntryListenerTxTest.class); - suite.addTestSuite(IgniteCacheEntryListenerTxReplicatedTest.class); - suite.addTestSuite(IgniteCacheEntryListenerTxLocalTest.class); - suite.addTestSuite(IgniteCacheEntryListenerEagerTtlDisabledTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerAtomicTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerAtomicReplicatedTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerAtomicLocalTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerTxTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerTxReplicatedTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerTxLocalTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryListenerEagerTtlDisabledTest.class, ignoredTests); - suite.addTestSuite(IgniteClientAffinityAssignmentSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteClientAffinityAssignmentSelfTest.class, ignoredTests); - suite.addTestSuite(IgniteCacheAtomicInvokeTest.class); - suite.addTestSuite(IgniteCacheAtomicNearEnabledInvokeTest.class); - suite.addTestSuite(IgniteCacheAtomicWithStoreInvokeTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalInvokeTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalWithStoreInvokeTest.class); - suite.addTestSuite(IgniteCacheTxInvokeTest.class); - suite.addTestSuite(CacheEntryProcessorNonSerializableTest.class); - suite.addTestSuite(CacheEntryProcessorExternalizableFailedTest.class); - suite.addTestSuite(IgniteCacheEntryProcessorCallTest.class); - suite.addTestSuite(IgniteCacheTxNearEnabledInvokeTest.class); - suite.addTestSuite(IgniteCacheTxLocalInvokeTest.class); - suite.addTestSuite(IgniteCrossCacheTxStoreSelfTest.class); - suite.addTestSuite(IgniteCacheEntryProcessorSequentialCallTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicNearEnabledInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicWithStoreInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicLocalInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicLocalWithStoreInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheTxInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheEntryProcessorNonSerializableTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheEntryProcessorExternalizableFailedTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryProcessorCallTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheTxNearEnabledInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheTxLocalInvokeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCrossCacheTxStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheEntryProcessorSequentialCallTest.class, ignoredTests); // TODO GG-11148: include test when implemented. // Test fails due to incorrect handling of CacheConfiguration#getCopyOnRead() and // CacheObjectContext#storeValue() properties. Heap storage should be redesigned in this ticket. //GridTestUtils.addTestIfNeeded(suite, CacheEntryProcessorCopySelfTest.class, ignoredTests); - suite.addTestSuite(IgnitePutAllLargeBatchSelfTest.class); - suite.addTestSuite(IgnitePutAllUpdateNonPreloadedPartitionSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, IgnitePutAllLargeBatchSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgnitePutAllUpdateNonPreloadedPartitionSelfTest.class, ignoredTests); // User's class loader tests. GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicExecutionContextTest.class, ignoredTests); - GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionedExecutionContextTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheReplicatedExecutionContextTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheTxExecutionContextTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheContinuousExecutionContextTest.class, ignoredTests); @@ -243,153 +212,150 @@ public static TestSuite suite(Set ignoredTests) throws Exception { GridTestUtils.addTestIfNeeded(suite, IgniteCacheSharedExecutionContextTest.class, ignoredTests); // Warmup closure tests. - suite.addTestSuite(IgniteWarmupClosureSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteWarmupClosureSelfTest.class, ignoredTests); // Swap tests. - suite.addTestSuite(GridCacheSwapPreloadSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheSwapPreloadSelfTest.class, ignoredTests); // Common tests. - suite.addTestSuite(CacheNamesSelfTest.class); - suite.addTestSuite(CacheNamesWithSpecialCharactersTest.class); - suite.addTestSuite(GridCacheConcurrentMapSelfTest.class); - suite.addTestSuite(GridCacheAffinityMapperSelfTest.class); - suite.addTestSuite(CacheAffinityCallSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheNamesSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheNamesWithSpecialCharactersTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheConcurrentMapSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAffinityMapperSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheAffinityCallSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheAffinityRoutingSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheMvccSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheMvccPartitionedSelfTest.class); - suite.addTestSuite(GridCacheMvccManagerSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheMvccPartitionedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheMvccManagerSelfTest.class, ignoredTests); // TODO GG-11141. - // suite.addTestSuite(GridCacheP2PUndeploySelfTest.class); - suite.addTestSuite(GridCacheConfigurationValidationSelfTest.class); - suite.addTestSuite(GridCacheConfigurationConsistencySelfTest.class); - suite.addTestSuite(GridDataStorageConfigurationConsistencySelfTest.class); - suite.addTestSuite(DataStorageConfigurationValidationTest.class); - suite.addTestSuite(GridCacheJdbcBlobStoreSelfTest.class); - suite.addTestSuite(GridCacheJdbcBlobStoreMultithreadedSelfTest.class); - suite.addTestSuite(JdbcTypesDefaultTransformerTest.class); - suite.addTestSuite(CacheJdbcPojoStoreTest.class); - suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerSelfTest.class); - suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.class); - suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest.class); - suite.addTestSuite(CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.class); - suite.addTestSuite(CacheJdbcPojoStoreMultitreadedSelfTest.class); - suite.addTestSuite(GridCacheBalancingStoreSelfTest.class); - suite.addTestSuite(GridCacheAffinityApiSelfTest.class); - suite.addTestSuite(GridCacheStoreValueBytesSelfTest.class); + // GridTestUtils.addTestIfNeeded(suite,GridCacheP2PUndeploySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheConfigurationValidationSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheConfigurationConsistencySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridDataStorageConfigurationConsistencySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStorageConfigurationValidationTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheJdbcBlobStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheJdbcBlobStoreMultithreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, JdbcTypesDefaultTransformerTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheJdbcPojoStoreTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheJdbcPojoStoreBinaryMarshallerSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinarySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheJdbcPojoStoreBinaryMarshallerWithSqlEscapeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheJdbcPojoStoreBinaryMarshallerStoreKeepBinaryWithSqlEscapeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheJdbcPojoStoreMultitreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheBalancingStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAffinityApiSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheStoreValueBytesSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorPersistenceSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, DataStreamerUpdateAfterLoadTest.class, ignoredTests); - suite.addTestSuite(DataStreamerMultiThreadedSelfTest.class); - suite.addTestSuite(DataStreamerMultinodeCreateCacheTest.class); - suite.addTestSuite(DataStreamerImplSelfTest.class); - suite.addTestSuite(DataStreamerTimeoutTest.class); - suite.addTestSuite(DataStreamerClientReconnectAfterClusterRestartTest.class); + GridTestUtils.addTestIfNeeded(suite, DataStreamerMultiThreadedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStreamerMultinodeCreateCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStreamerImplSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStreamerTimeoutTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, DataStreamerClientReconnectAfterClusterRestartTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheEntryMemorySizeSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheClearAllSelfTest.class); - suite.addTestSuite(GridCacheObjectToStringSelfTest.class); - suite.addTestSuite(GridCacheLoadOnlyStoreAdapterSelfTest.class); - suite.addTestSuite(GridCacheGetStoreErrorSelfTest.class); - suite.addTestSuite(StoreResourceInjectionSelfTest.class); - suite.addTestSuite(CacheFutureExceptionSelfTest.class); - suite.addTestSuite(GridCacheAsyncOperationsLimitSelfTest.class); - suite.addTestSuite(IgniteCacheManyAsyncOperationsTest.class); - suite.addTestSuite(GridCacheTtlManagerSelfTest.class); - // TODO: ignite-4534 -// suite.addTestSuite(GridCacheTtlManagerEvictionSelfTest.class); - suite.addTestSuite(GridCacheLifecycleAwareSelfTest.class); - suite.addTestSuite(IgniteCacheAtomicStopBusySelfTest.class); - suite.addTestSuite(IgniteCacheTransactionalStopBusySelfTest.class); - suite.addTestSuite(GridCacheAtomicNearCacheSelfTest.class); - suite.addTestSuite(CacheAtomicNearUpdateTopologyChangeTest.class); - suite.addTestSuite(CacheTxNearUpdateTopologyChangeTest.class); - suite.addTestSuite(GridCacheStorePutxSelfTest.class); - suite.addTestSuite(GridCacheOffHeapMultiThreadedUpdateSelfTest.class); - suite.addTestSuite(GridCacheOffHeapAtomicMultiThreadedUpdateSelfTest.class); - suite.addTestSuite(GridCacheColocatedTxStoreExceptionSelfTest.class); - suite.addTestSuite(GridCacheReplicatedTxStoreExceptionSelfTest.class); - suite.addTestSuite(GridCacheLocalTxStoreExceptionSelfTest.class); - suite.addTestSuite(GridCacheNearTxStoreExceptionSelfTest.class); - suite.addTestSuite(GridCacheMissingCommitVersionSelfTest.class); - suite.addTestSuite(GridCacheEntrySetIterationPreloadingSelfTest.class); - suite.addTestSuite(GridCacheMixedPartitionExchangeSelfTest.class); - suite.addTestSuite(IgniteCacheAtomicMessageRecoveryTest.class); - suite.addTestSuite(IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.class); - suite.addTestSuite(IgniteCacheAtomicMessageRecovery10ConnectionsTest.class); - suite.addTestSuite(IgniteCacheTxMessageRecoveryTest.class); - suite.addTestSuite(IgniteCacheMessageWriteTimeoutTest.class); - suite.addTestSuite(IgniteCacheMessageRecoveryIdleConnectionTest.class); - suite.addTestSuite(IgniteCacheConnectionRecoveryTest.class); - suite.addTestSuite(IgniteCacheConnectionRecovery10ConnectionsTest.class); - suite.addTestSuite(GridCacheGlobalLoadTest.class); - suite.addTestSuite(GridCachePartitionedLocalStoreSelfTest.class); - suite.addTestSuite(GridCacheReplicatedLocalStoreSelfTest.class); - suite.addTestSuite(GridCacheTxPartitionedLocalStoreSelfTest.class); - suite.addTestSuite(IgniteCacheSystemTransactionsSelfTest.class); - suite.addTestSuite(CacheDeferredDeleteSanitySelfTest.class); - suite.addTestSuite(CacheDeferredDeleteQueueTest.class); - suite.addTestSuite(GridCachePartitionsStateValidatorSelfTest.class); - suite.addTestSuite(GridCachePartitionsStateValidationTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheClearAllSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheObjectToStringSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLoadOnlyStoreAdapterSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheGetStoreErrorSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, StoreResourceInjectionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheFutureExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAsyncOperationsLimitSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheManyAsyncOperationsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheTtlManagerSelfTest.class, ignoredTests); +// GridTestUtils.addTestIfNeeded(suite, GridCacheTtlManagerEvictionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLifecycleAwareSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicStopBusySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheTransactionalStopBusySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicNearCacheSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheAtomicNearUpdateTopologyChangeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheTxNearUpdateTopologyChangeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedStorePutSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapMultiThreadedUpdateSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapAtomicMultiThreadedUpdateSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheColocatedTxStoreExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheReplicatedTxStoreExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalTxStoreExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearTxStoreExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheMissingCommitVersionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEntrySetIterationPreloadingSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheMixedPartitionExchangeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicMessageRecoveryTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicMessageRecoveryPairedConnectionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheAtomicMessageRecovery10ConnectionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheTxMessageRecoveryTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheMessageWriteTimeoutTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheMessageRecoveryIdleConnectionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheConnectionRecoveryTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheConnectionRecovery10ConnectionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheGlobalLoadTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedLocalStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheReplicatedLocalStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheTxPartitionedLocalStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheSystemTransactionsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheDeferredDeleteSanitySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheDeferredDeleteQueueTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionsStateValidatorSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionsStateValidationTest.class, ignoredTests); - suite.addTest(IgniteCacheTcpClientDiscoveryTestSuite.suite()); + suite.addTest(IgniteCacheTcpClientDiscoveryTestSuite.suite(ignoredTests)); // Heuristic exception handling. - suite.addTestSuite(GridCacheColocatedTxExceptionSelfTest.class); - suite.addTestSuite(GridCacheReplicatedTxExceptionSelfTest.class); - suite.addTestSuite(GridCacheLocalTxExceptionSelfTest.class); - suite.addTestSuite(GridCacheNearTxExceptionSelfTest.class); - suite.addTestSuite(GridCacheStopSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheColocatedTxExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheReplicatedTxExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalTxExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearTxExceptionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheStopSelfTest.class, ignoredTests); - suite.addTestSuite(IgniteCacheNearLockValueSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheNearLockValueSelfTest.class, ignoredTests); - suite.addTestSuite(CachePutEventListenerErrorSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, CachePutEventListenerErrorSelfTest.class, ignoredTests); - suite.addTestSuite(IgniteTxConfigCacheSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteTxConfigCacheSelfTest.class, ignoredTests); - suite.addTestSuite(CacheTxFastFinishTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheTxFastFinishTest.class, ignoredTests); - //suite.addTestSuite(GridIoManagerSelfTest.class); - suite.addTestSuite(IgniteVariousConnectionNumberTest.class); - suite.addTestSuite(IgniteCommunicationBalanceTest.class); - suite.addTestSuite(IgniteCommunicationBalancePairedConnectionsTest.class); - suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class); - suite.addTestSuite(IgniteCommunicationSslBalanceTest.class); - suite.addTestSuite(IgniteIoTestMessagesTest.class); - suite.addTestSuite(IgniteDiagnosticMessagesTest.class); - suite.addTestSuite(IgniteDiagnosticMessagesMultipleConnectionsTest.class); + //GridTestUtils.addTestIfNeeded(suite,GridIoManagerSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteVariousConnectionNumberTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCommunicationBalanceTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCommunicationBalancePairedConnectionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCommunicationBalanceMultipleConnectionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCommunicationSslBalanceTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteIoTestMessagesTest.class, ignoredTests); - suite.addTestSuite(IgniteIncompleteCacheObjectSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteIncompleteCacheObjectSelfTest.class, ignoredTests); - suite.addTestSuite(GridStoreLoadCacheTest.class); - suite.addTestSuite(CacheStoreReadFromBackupTest.class); - suite.addTestSuite(CacheStoreWriteErrorTest.class); - suite.addTestSuite(CacheTransactionalStoreReadFromBackupTest.class); + GridTestUtils.addTestIfNeeded(suite, GridStoreLoadCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheStoreReadFromBackupTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheStoreWriteErrorTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheTransactionalStoreReadFromBackupTest.class, ignoredTests); - //suite.addTestSuite(CacheAtomicSingleMessageCountSelfTest.class); - //suite.addTestSuite(GridCacheAtomicUsersAffinityMapperSelfTest.class); - //suite.addTestSuite(GridCacheClearLocallySelfTest.class); - //suite.addTestSuite(GridCacheConcurrentGetCacheOnClientTest.class); - //suite.addTestSuite(GridCacheFullTextQueryMultithreadedSelfTest.class); - //suite.addTestSuite(GridCacheKeyCheckNearEnabledSelfTest.class); - //suite.addTestSuite(GridCacheKeyCheckSelfTest.class); - //suite.addTestSuite(GridCacheLeakTest.class); - //suite.addTestSuite(GridCacheMultiUpdateLockSelfTest.class); - //suite.addTestSuite(GridCacheMvccFlagsTest.class); - //suite.addTestSuite(GridCacheReplicatedUsersAffinityMapperSelfTest.class); - //suite.addTestSuite(GridCacheReturnValueTransferSelfTest.class); - //suite.addTestSuite(GridCacheSlowTxWarnTest.class); - //suite.addTestSuite(GridCacheTtlManagerLoadTest.class); - //suite.addTestSuite(GridCacheTxUsersAffinityMapperSelfTest.class); - //suite.addTestSuite(IgniteInternalCacheRemoveTest.class); - //suite.addTestSuite(IgniteCacheBinaryEntryProcessorSelfTest.class); - //suite.addTestSuite(IgniteCacheObjectPutSelfTest.class); - //suite.addTestSuite(IgniteCacheSerializationSelfTest.class); - //suite.addTestSuite(IgniteCacheStartStopLoadTest.class); - //suite.addTestSuite(IgniteCachingProviderSelfTest.class); - //suite.addTestSuite(IgniteOnePhaseCommitNearSelfTest.class); - //suite.addTestSuite(IgniteStaticCacheStartSelfTest.class); - //suite.addTestSuite(InterceptorWithKeepBinaryCacheFullApiTest.class); + //GridTestUtils.addTestIfNeeded(suite,CacheAtomicSingleMessageCountSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheAtomicUsersAffinityMapperSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheClearLocallySelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheConcurrentGetCacheOnClientTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheFullTextQueryMultithreadedSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheKeyCheckNearEnabledSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheKeyCheckSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheLeakTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheMultiUpdateLockSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheMvccFlagsTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReplicatedUsersAffinityMapperSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheReturnValueTransferSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheSlowTxWarnTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheTtlManagerLoadTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,GridCacheTxUsersAffinityMapperSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteInternalCacheRemoveTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheBinaryEntryProcessorSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheObjectPutSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheSerializationSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCacheStartStopLoadTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteCachingProviderSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteOnePhaseCommitNearSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,IgniteStaticCacheStartSelfTest.class, ignoredTests); + //GridTestUtils.addTestIfNeeded(suite,InterceptorWithKeepBinaryCacheFullApiTest.class, ignoredTests); - suite.addTestSuite(BinaryMetadataRegistrationInsideEntryProcessorTest.class); + GridTestUtils.addTestIfNeeded(suite, BinaryMetadataRegistrationInsideEntryProcessorTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index e367aad4e4930..bf2aa9f6234b4 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -202,7 +202,6 @@ public static TestSuite suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedAtomicGetAndTransformStoreSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedBasicStoreSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridNearCacheStoreUpdateTest.class, ignoredTests); - //GridTestUtils.addTestIfNeeded(suite,GridCachePartitionedStorePutSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedBasicStoreMultiNodeSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedNearDisabledBasicStoreMultiNodeSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheConcurrentReadThroughTest.class, ignoredTests); From d59f4d5c0897c58a8adcff8fdb2492c0f3cf68c9 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Tue, 11 Dec 2018 14:12:21 +0300 Subject: [PATCH 274/403] IGNITE-10483: MVCC: Enlist requests failure handling. This closes #5554. (cherry picked from commit 33e6135c846053031565e31de98bd891093caf1a) --- .../processors/cache/GridCacheIoManager.java | 68 +++++++++++++++++++ .../dht/GridDhtTxQueryEnlistResponse.java | 2 +- ...ntryProcessorExternalizableFailedTest.java | 15 ++-- ...acheEntryProcessorNonSerializableTest.java | 7 ++ 4 files changed, 83 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java index 23cd4ba1afec9..87929e50fab49 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java @@ -51,6 +51,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxQueryEnlistRequest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxQueryEnlistResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicNearResponse; @@ -73,10 +75,16 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetResponse; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistResponse; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishResponse; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryEnlistRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryEnlistResponse; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryResultsEnlistRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryResultsEnlistResponse; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryRequest; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryResponse; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxState; @@ -1017,6 +1025,66 @@ private void processFailedMessage(UUID nodeId, break; + case 151: { + GridNearTxQueryEnlistRequest req = (GridNearTxQueryEnlistRequest)msg; + + GridNearTxQueryEnlistResponse res = new GridNearTxQueryEnlistResponse( + req.cacheId(), + req.futureId(), + req.miniId(), + req.version(), + req.classError()); + + sendResponseOnFailedMessage(nodeId, res, cctx, plc); + + break; + } + + case 153: { + GridNearTxQueryResultsEnlistRequest req = (GridNearTxQueryResultsEnlistRequest)msg; + + GridNearTxQueryEnlistResponse res = new GridNearTxQueryResultsEnlistResponse( + req.cacheId(), + req.futureId(), + req.miniId(), + req.version(), + req.classError()); + + sendResponseOnFailedMessage(nodeId, res, cctx, plc); + + break; + } + + case 155: /* GridDhtTxQueryEnlistRequest */ + case 156: /* GridDhtTxQueryFirstEnlistRequest */ { + GridDhtTxQueryEnlistRequest req = (GridDhtTxQueryEnlistRequest)msg; + + GridDhtTxQueryEnlistResponse res = new GridDhtTxQueryEnlistResponse( + req.cacheId(), + req.dhtFutureId(), + req.batchId(), + req.classError()); + + sendResponseOnFailedMessage(nodeId, res, cctx, plc); + + break; + } + + case 159: { + GridNearTxEnlistRequest req = (GridNearTxEnlistRequest)msg; + + GridNearTxEnlistResponse res = new GridNearTxEnlistResponse( + req.cacheId(), + req.futureId(), + req.miniId(), + req.version(), + req.classError()); + + sendResponseOnFailedMessage(nodeId, res, cctx, plc); + + break; + } + case -36: { GridDhtAtomicSingleUpdateRequest req = (GridDhtAtomicSingleUpdateRequest)msg; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java index 18f4baa52808d..42554362199aa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistResponse.java @@ -60,7 +60,7 @@ public GridDhtTxQueryEnlistResponse() { * @param batchId Batch id. * @param err Error. */ - GridDhtTxQueryEnlistResponse(int cacheId, IgniteUuid futId, int batchId, + public GridDhtTxQueryEnlistResponse(int cacheId, IgniteUuid futId, int batchId, Throwable err) { this.cacheId = cacheId; this.futId = futId; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java index 6e4a9e5c77526..4ca4b75726ca6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorExternalizableFailedTest.java @@ -105,6 +105,13 @@ public class CacheEntryProcessorExternalizableFailedTest extends GridCommonAbstr startGrid(getServerNodeCount()); } + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -467,8 +474,6 @@ public void testPessimisticFullSyncWithNearCache() throws Exception { * @throws Exception If failed. */ public void testMvccPessimisticOnePhaseCommit() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); @@ -498,8 +503,6 @@ public void testMvccPessimisticOnePhaseCommitWithNearCache() throws Exception { * @throws Exception If failed. */ public void testMvccPessimisticOnePhaseCommitFullSync() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 1).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); @@ -529,8 +532,6 @@ public void testMvccPessimisticOnePhaseCommitFullSyncWithNearCache() throws Exce * @throws Exception If failed. */ public void testMvccPessimistic() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - CacheConfiguration ccfg = cacheConfiguration(PRIMARY_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); @@ -560,8 +561,6 @@ public void testMvccPessimisticWithNearCache() throws Exception { * @throws Exception If failed. */ public void testMvccPessimisticFullSync() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10483"); - CacheConfiguration ccfg = cacheConfiguration(FULL_SYNC, 2).setAtomicityMode(TRANSACTIONAL_SNAPSHOT); doTestInvokeTest(ccfg, PESSIMISTIC, REPEATABLE_READ); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java index ad03bf5a91199..25ae1d1900808 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheEntryProcessorNonSerializableTest.java @@ -102,6 +102,13 @@ public class CacheEntryProcessorNonSerializableTest extends GridCommonAbstractTe startGrid(getServerNodeCount()); } + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + /** * @return Server nodes. */ From d06640b4051dba389d9f44c77776a6bd53a143ff Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Wed, 5 Dec 2018 16:26:35 +0300 Subject: [PATCH 275/403] IGNITE-10368: MVCC: Create "Cache 9" test suite for MVCC mode. This closes #5556. (cherry picked from commit bc895ac62f8049ac52f007cf80b23d1296198ad9) --- .../cache/CachePutIfAbsentTest.java | 4 + .../IgniteCachePrimarySyncTest.java | 53 ++++++---- .../IgniteTxCachePrimarySyncTest.java | 96 ++++++++++++++++--- ...SynchronizationModesMultithreadedTest.java | 78 ++++++++------- .../IgniteTxConcurrentRemoveObjectsTest.java | 52 +++++++--- .../testsuites/IgniteCacheMvccTestSuite9.java | 57 +++++++++++ .../testsuites/IgniteCacheTestSuite9.java | 31 +++--- 7 files changed, 282 insertions(+), 89 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutIfAbsentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutIfAbsentTest.java index 00ba25fb7651c..a70d3c3f18efa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutIfAbsentTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CachePutIfAbsentTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -129,6 +130,9 @@ public void testTxConflictGetAndPutIfAbsent() throws Exception { for (TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + try (Transaction tx = txs.txStart(concurrency, isolation)) { Object old = cache.getAndPutIfAbsent(key, 3); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java index e9e22ee90429f..2482a8ef46e0c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePrimarySyncTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -35,6 +36,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; @@ -52,6 +54,15 @@ public class IgniteCachePrimarySyncTest extends GridCommonAbstractTest { /** */ private static final int SRVS = 4; + /** */ + private static final String ATOMIC_CACHE = "atomicCache"; + + /** */ + private static final String TX_CACHE = "txCache"; + + /** */ + private static final String MVCC_CACHE = "mvccCache"; + /** */ private boolean clientMode; @@ -61,19 +72,22 @@ public class IgniteCachePrimarySyncTest extends GridCommonAbstractTest { ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder); - CacheConfiguration ccfg1 = new CacheConfiguration<>(DEFAULT_CACHE_NAME); - ccfg1.setName("cache1"); - ccfg1.setAtomicityMode(ATOMIC); - ccfg1.setBackups(2); - ccfg1.setWriteSynchronizationMode(PRIMARY_SYNC); + CacheConfiguration ccfg1 = new CacheConfiguration<>(ATOMIC_CACHE) + .setAtomicityMode(ATOMIC) + .setBackups(2) + .setWriteSynchronizationMode(PRIMARY_SYNC); + + CacheConfiguration ccfg2 = new CacheConfiguration<>(TX_CACHE) + .setAtomicityMode(TRANSACTIONAL) + .setBackups(2) + .setWriteSynchronizationMode(PRIMARY_SYNC); - CacheConfiguration ccfg2 = new CacheConfiguration<>(DEFAULT_CACHE_NAME); - ccfg2.setName("cache2"); - ccfg2.setAtomicityMode(TRANSACTIONAL); - ccfg2.setBackups(2); - ccfg2.setWriteSynchronizationMode(PRIMARY_SYNC); + CacheConfiguration ccfg3 = new CacheConfiguration<>(MVCC_CACHE) + .setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setBackups(2) + .setWriteSynchronizationMode(PRIMARY_SYNC); - cfg.setCacheConfiguration(ccfg1, ccfg2); + cfg.setCacheConfiguration(ccfg1, ccfg2, ccfg3); cfg.setClientMode(clientMode); @@ -97,17 +111,24 @@ public class IgniteCachePrimarySyncTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testPutGet() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10520"); + Ignite ignite = ignite(SRVS); - checkPutGet(ignite.cache("cache1"), null, null, null); + checkPutGet(ignite.cache(ATOMIC_CACHE), null, null, null); + + checkPutGet(ignite.cache(TX_CACHE), null, null, null); + + checkPutGet(ignite.cache(MVCC_CACHE), null, null, null); - checkPutGet(ignite.cache("cache2"), null, null, null); + checkPutGet(ignite.cache(TX_CACHE), ignite.transactions(), OPTIMISTIC, REPEATABLE_READ); - checkPutGet(ignite.cache("cache2"), ignite.transactions(), OPTIMISTIC, REPEATABLE_READ); + checkPutGet(ignite.cache(TX_CACHE), ignite.transactions(), OPTIMISTIC, SERIALIZABLE); - checkPutGet(ignite.cache("cache2"), ignite.transactions(), OPTIMISTIC, SERIALIZABLE); + checkPutGet(ignite.cache(TX_CACHE), ignite.transactions(), PESSIMISTIC, READ_COMMITTED); - checkPutGet(ignite.cache("cache2"), ignite.transactions(), PESSIMISTIC, READ_COMMITTED); + checkPutGet(ignite.cache(MVCC_CACHE), ignite.transactions(), PESSIMISTIC, REPEATABLE_READ); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java index bdf0b12dee21d..ceed9b868cf36 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCachePrimarySyncTest.java @@ -58,6 +58,8 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; +import org.apache.ignite.testframework.MvccFeatureChecker.Feature; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -134,6 +136,9 @@ public class IgniteTxCachePrimarySyncTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testSingleKeyCommitFromPrimary() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10518"); + singleKeyCommitFromPrimary(cacheConfiguration(DEFAULT_CACHE_NAME, PRIMARY_SYNC, 1, true, false)); singleKeyCommitFromPrimary(cacheConfiguration(DEFAULT_CACHE_NAME, PRIMARY_SYNC, 2, false, false)); @@ -148,6 +153,16 @@ public void testSingleKeyCommitFromPrimary() throws Exception { * @throws Exception If failed. */ private void singleKeyCommitFromPrimary(CacheConfiguration ccfg) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + if (ccfg.getCacheStoreFactory() != null && + !MvccFeatureChecker.isSupported(Feature.CACHE_STORE)) + return; + + if (ccfg.getNearConfiguration() != null && + !MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) + return; + } + Ignite ignite = ignite(0); IgniteCache cache = ignite.createCache(ccfg); @@ -164,6 +179,9 @@ private void singleKeyCommitFromPrimary(CacheConfiguration ccfg) for (final TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (final TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + singleKeyCommitFromPrimary(node, ccfg, new IgniteBiInClosure>() { @Override public void apply(Integer key, IgniteCache cache) { Ignite ignite = cache.unwrap(Ignite.class); @@ -266,12 +284,19 @@ public void testSingleKeyPrimaryNodeFail2() throws Exception { * @throws Exception If failed. */ private void singleKeyPrimaryNodeLeft(CacheConfiguration ccfg) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + if (ccfg.getCacheStoreFactory() != null && + !MvccFeatureChecker.isSupported(Feature.CACHE_STORE)) + return; + } + Ignite ignite = ignite(0); IgniteCache cache = ignite.createCache(ccfg); try { - ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) + ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); for (int i = 0; i < NODES; i++) { Ignite node = ignite(i); @@ -284,6 +309,9 @@ private void singleKeyPrimaryNodeLeft(CacheConfiguration ccfg) t for (final TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (final TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + singleKeyPrimaryNodeLeft(node, ccfg, new IgniteBiInClosure>() { @Override public void apply(Integer key, IgniteCache cache) { Ignite ignite = cache.unwrap(Ignite.class); @@ -372,6 +400,9 @@ private void singleKeyPrimaryNodeLeft( * @throws Exception If failed. */ public void testSingleKeyCommit() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10518"); + singleKeyCommit(cacheConfiguration(DEFAULT_CACHE_NAME, PRIMARY_SYNC, 1, true, false)); singleKeyCommit(cacheConfiguration(DEFAULT_CACHE_NAME, PRIMARY_SYNC, 2, false, false)); @@ -386,12 +417,23 @@ public void testSingleKeyCommit() throws Exception { * @throws Exception If failed. */ private void singleKeyCommit(CacheConfiguration ccfg) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + if (ccfg.getCacheStoreFactory() != null && + !MvccFeatureChecker.isSupported(Feature.CACHE_STORE)) + return; + + if (ccfg.getNearConfiguration() != null && + !MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) + return; + } + Ignite ignite = ignite(0); IgniteCache cache = ignite.createCache(ccfg); try { - ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) + ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); for (int i = 1; i < NODES; i++) { Ignite node = ignite(i); @@ -406,6 +448,9 @@ private void singleKeyCommit(CacheConfiguration ccfg) throws Exc for (final TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (final TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + singleKeyCommit(node, ccfg, new IgniteBiInClosure>() { @Override public void apply(Integer key, IgniteCache cache) { Ignite ignite = cache.unwrap(Ignite.class); @@ -528,12 +573,23 @@ public void testWaitPrimaryResponse() throws Exception { * @throws Exception If failed. */ private void checkWaitPrimaryResponse(CacheConfiguration ccfg) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + if (ccfg.getCacheStoreFactory() != null && + !MvccFeatureChecker.isSupported(Feature.CACHE_STORE)) + return; + + if (ccfg.getNearConfiguration() != null && + !MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) + return; + } + Ignite ignite = ignite(0); IgniteCache cache = ignite.createCache(ccfg); try { - ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) + ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); for (int i = 1; i < NODES; i++) { Ignite node = ignite(i); @@ -561,6 +617,9 @@ private void checkWaitPrimaryResponse(CacheConfiguration ccfg) t for (final TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (final TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + checkWaitPrimaryResponse(node, ccfg, new IgniteBiInClosure>() { @Override public void apply(Integer key, IgniteCache cache) { Ignite ignite = cache.unwrap(Ignite.class); @@ -652,6 +711,9 @@ private void checkWaitPrimaryResponse( * @throws Exception If failed. */ public void testOnePhaseMessages() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + return; // Not supported. Commit flow differs for Mvcc mode. + checkOnePhaseMessages(cacheConfiguration(DEFAULT_CACHE_NAME, PRIMARY_SYNC, 1, false, false)); } @@ -678,6 +740,9 @@ private void checkOnePhaseMessages(CacheConfiguration ccfg) thro for (final TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (final TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + checkOnePhaseMessages(node, ccfg, new IgniteBiInClosure>() { @Override public void apply(Integer key, IgniteCache cache) { Ignite ignite = cache.unwrap(Ignite.class); @@ -760,12 +825,13 @@ public void testTxSyncMode() throws Exception { List> caches = new ArrayList<>(); try { - caches.add(createCache(ignite, cacheConfiguration("fullSync1", FULL_SYNC, 1, false, false), true)); - caches.add(createCache(ignite, cacheConfiguration("fullSync2", FULL_SYNC, 1, false, false), true)); - caches.add(createCache(ignite, cacheConfiguration("fullAsync1", FULL_ASYNC, 1, false, false), true)); - caches.add(createCache(ignite, cacheConfiguration("fullAsync2", FULL_ASYNC, 1, false, false), true)); - caches.add(createCache(ignite, cacheConfiguration("primarySync1", PRIMARY_SYNC, 1, false, false), true)); - caches.add(createCache(ignite, cacheConfiguration("primarySync2", PRIMARY_SYNC, 1, false, false), true)); + + caches.add(createCache(ignite, cacheConfiguration("fullSync1", FULL_SYNC, 1, false, false))); + caches.add(createCache(ignite, cacheConfiguration("fullSync2", FULL_SYNC, 1, false, false))); + caches.add(createCache(ignite, cacheConfiguration("fullAsync1", FULL_ASYNC, 1, false, false))); + caches.add(createCache(ignite, cacheConfiguration("fullAsync2", FULL_ASYNC, 1, false, false))); + caches.add(createCache(ignite, cacheConfiguration("primarySync1", PRIMARY_SYNC, 1, false, false))); + caches.add(createCache(ignite, cacheConfiguration("primarySync2", PRIMARY_SYNC, 1, false, false))); for (int i = 0; i < NODES; i++) { checkTxSyncMode(ignite(i), true); @@ -805,7 +871,8 @@ private void waitKeyRemoved(final String cacheName, final Object key) throws Exc * @param key Cache key. * @throws Exception If failed. */ - private void waitKeyUpdated(Ignite ignite, int expNodes, final String cacheName, final Object key) throws Exception { + private void waitKeyUpdated(Ignite ignite, int expNodes, final String cacheName, + final Object key) throws Exception { Affinity aff = ignite.affinity(cacheName); final Collection nodes = aff.mapKeyToPrimaryAndBackups(key); @@ -834,14 +901,12 @@ private void waitKeyUpdated(Ignite ignite, int expNodes, final String cacheName, /** * @param ignite Node. * @param ccfg Cache configuration. - * @param nearCache If {@code true} creates near cache on one of client nodes. * @return Created cache. */ - private IgniteCache createCache(Ignite ignite, CacheConfiguration ccfg, - boolean nearCache) { + private IgniteCache createCache(Ignite ignite, CacheConfiguration ccfg) { IgniteCache cache = ignite.createCache(ccfg); - if (nearCache) + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(Feature.NEAR_CACHE)) ignite(NODES - 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>()); return cache; @@ -866,6 +931,9 @@ private void checkTxSyncMode(Ignite ignite, boolean commit) { for (TransactionConcurrency concurrency : TransactionConcurrency.values()) { for (TransactionIsolation isolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + try (Transaction tx = txs.txStart(concurrency, isolation)) { fullSync1.put(key++, 1); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java index bed8a41149a80..09a93cd0e4a4b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxCacheWriteSynchronizationModesMultithreadedTest.java @@ -32,7 +32,6 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.store.CacheStore; import org.apache.ignite.cache.store.CacheStoreAdapter; @@ -49,6 +48,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionOptimisticException; @@ -105,24 +105,26 @@ public class IgniteTxCacheWriteSynchronizationModesMultithreadedTest extends Gri /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); - System.setProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL,"true"); + super.beforeTestsStarted(); - startGrids(SRVS); + startGridsMultiThreaded(SRVS); clientMode = true; - for (int i = 0; i < CLIENTS; i++) { - Ignite client = startGrid(SRVS + i); + startGridsMultiThreaded(SRVS, CLIENTS); - assertTrue(client.configuration().isClientMode()); - } + for (int i = 0; i < CLIENTS; i++) + assertTrue(grid(SRVS + i).configuration().isClientMode()); } /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { - System.clearProperty(IgniteSystemProperties.IGNITE_ENABLE_FORCIBLE_NODE_KILL); + stopAllGrids(); + + super.afterTestsStopped(); } /** @@ -195,6 +197,14 @@ private void multithreaded(CacheWriteSynchronizationMode syncMode, boolean store, boolean nearCache, boolean restart) throws Exception { + if (MvccFeatureChecker.forcedMvcc()) { + if (store && !MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + return; + + if (nearCache && !MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) + return; + } + final Ignite ignite = ignite(0); createCache(ignite, cacheConfiguration(DEFAULT_CACHE_NAME, syncMode, backups, store), nearCache); @@ -271,36 +281,38 @@ private void multithreaded(CacheWriteSynchronizationMode syncMode, } }); - commitMultithreaded(new IgniteBiInClosure>() { - @Override public void apply(Ignite ignite, IgniteCache cache) { - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - - Map map = new LinkedHashMap<>(); - - for (int i = 0; i < 10; i++) { - Integer key = rnd.nextInt(MULTITHREADED_TEST_KEYS); - - map.put(key, rnd.nextInt()); - } + if (!MvccFeatureChecker.forcedMvcc()) { + commitMultithreaded(new IgniteBiInClosure>() { + @Override public void apply(Ignite ignite, IgniteCache cache) { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); - while (true) { - try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) { - for (Map.Entry e : map.entrySet()) - cache.put(e.getKey(), e.getValue()); + Map map = new LinkedHashMap<>(); - tx.commit(); + for (int i = 0; i < 10; i++) { + Integer key = rnd.nextInt(MULTITHREADED_TEST_KEYS); - break; - } - catch (TransactionOptimisticException ignored) { - // Retry. + map.put(key, rnd.nextInt()); } - catch (CacheException | IgniteException ignored) { - break; + + while (true) { + try (Transaction tx = ignite.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) { + for (Map.Entry e : map.entrySet()) + cache.put(e.getKey(), e.getValue()); + + tx.commit(); + + break; + } + catch (TransactionOptimisticException ignored) { + // Retry. + } + catch (CacheException | IgniteException ignored) { + break; + } } } - } - }); + }); + } } finally { stop.set(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java index a5b2cb71a4ef9..cd5cbe6b84d60 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxConcurrentRemoveObjectsTest.java @@ -36,9 +36,11 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL; -import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; /** @@ -67,25 +69,22 @@ public class IgniteTxConcurrentRemoveObjectsTest extends GridCommonAbstractTest oldIgniteCacheRmvEntriesTtl = Long.getLong(IGNITE_CACHE_REMOVED_ENTRIES_TTL, 10_000); System.setProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL, Long.toString(newIgniteCacheRemovedEntriesTtl)); + + startGrid(0); } /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + System.setProperty(IGNITE_CACHE_REMOVED_ENTRIES_TTL, Long.toString(oldIgniteCacheRmvEntriesTtl)); super.afterTestsStopped(); } - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - stopAllGrids(); - } - /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { - stopAllGrids(); + grid(0).destroyCache(DEFAULT_CACHE_NAME); super.afterTest(); } @@ -102,7 +101,7 @@ public class IgniteTxConcurrentRemoveObjectsTest extends GridCommonAbstractTest /** * @return Cache configuration. */ - private CacheConfiguration getCacheCfg() { + private CacheConfiguration cacheConfiguration() { CacheConfiguration ccfg = new CacheConfiguration<>(); ccfg.setName(DEFAULT_CACHE_NAME); @@ -114,6 +113,28 @@ private CacheConfiguration getCacheCfg() { return ccfg; } + /** + * @throws Exception If failed. + */ + public void testOptimisticTxLeavesObjectsInLocalPartition() throws Exception { + checkTxLeavesObjectsInLocalPartition(cacheConfiguration(), TransactionConcurrency.OPTIMISTIC, SERIALIZABLE); + } + + /** + * @throws Exception If failed. + */ + public void testPessimisticTxLeavesObjectsInLocalPartition() throws Exception { + checkTxLeavesObjectsInLocalPartition(cacheConfiguration(), TransactionConcurrency.PESSIMISTIC, SERIALIZABLE); + } + + /** + * @throws Exception If failed. + */ + public void testMvccTxLeavesObjectsInLocalPartition() throws Exception { + checkTxLeavesObjectsInLocalPartition(cacheConfiguration().setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT), + TransactionConcurrency.PESSIMISTIC, REPEATABLE_READ); + } + /** * Too many deletes in single transaction may overflow {@link GridDhtLocalPartition#rmvQueue} and entries will be * deleted synchronously in {@link GridDhtLocalPartition#onDeferredDelete(int, KeyCacheObject, GridCacheVersion)}. @@ -121,10 +142,11 @@ private CacheConfiguration getCacheCfg() { * * @throws Exception If failed. */ - public void testTxLeavesObjectsInLocalPartition() throws Exception { - IgniteEx igniteEx = startGrid(getConfiguration()); + public void checkTxLeavesObjectsInLocalPartition(CacheConfiguration ccfg, + TransactionConcurrency optimistic, TransactionIsolation isolation) throws Exception { + IgniteEx igniteEx = grid(0); - igniteEx.getOrCreateCache(getCacheCfg()); + igniteEx.getOrCreateCache(ccfg); try (IgniteDataStreamer dataStreamer = igniteEx.dataStreamer(DEFAULT_CACHE_NAME)) { for (int i = 0; i < CACHE_ENTRIES_COUNT; i++) @@ -141,8 +163,8 @@ public void testTxLeavesObjectsInLocalPartition() throws Exception { assertEquals(CACHE_ENTRIES_COUNT, client.getOrCreateCache(DEFAULT_CACHE_NAME).size()); - try (Transaction tx = client.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) { - IgniteCache cache = client.getOrCreateCache(getCacheCfg()); + try (Transaction tx = client.transactions().txStart(optimistic, isolation)) { + IgniteCache cache = client.getOrCreateCache(cacheConfiguration()); for (int v = 0; v < CACHE_ENTRIES_COUNT; v++) { cache.get(v); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java new file mode 100644 index 0000000000000..17a9421a1a34c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.Collection; +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; + +/** + * Test suite. + */ +public class IgniteCacheMvccTestSuite9 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + Collection ignoredTests = new HashSet<>(); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(IgniteTxConcurrentRemoveObjectsTest.class); + + // Atomic caches. + ignoredTests.add(CacheAtomicPrimarySyncBackPressureTest.class); + + // Other non-tx tests. + ignoredTests.add(IgniteCacheGetCustomCollectionsSelfTest.class); + ignoredTests.add(IgniteCacheLoadRebalanceEvictionSelfTest.class); + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 9"); + + suite.addTest(IgniteCacheTestSuite9.suite(ignoredTests)); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index b8da01e4e9a08..5aacada45e06c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.CachePutIfAbsentTest; import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest; @@ -27,6 +28,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; import org.apache.ignite.internal.processors.cache.transactions.TxDataConsistencyOnCommitFailureTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -34,23 +36,30 @@ public class IgniteCacheTestSuite9 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 9"); - suite.addTestSuite(IgniteCacheGetCustomCollectionsSelfTest.class); - suite.addTestSuite(IgniteCacheLoadRebalanceEvictionSelfTest.class); - suite.addTestSuite(IgniteCachePrimarySyncTest.class); - suite.addTestSuite(IgniteTxCachePrimarySyncTest.class); - suite.addTestSuite(IgniteTxCacheWriteSynchronizationModesMultithreadedTest.class); - suite.addTestSuite(CachePutIfAbsentTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheGetCustomCollectionsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheLoadRebalanceEvictionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCachePrimarySyncTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTxCachePrimarySyncTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTxCacheWriteSynchronizationModesMultithreadedTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CachePutIfAbsentTest.class, ignoredTests); - suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheAtomicPrimarySyncBackPressureTest.class, ignoredTests); - suite.addTestSuite(IgniteTxConcurrentRemoveObjectsTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteTxConcurrentRemoveObjectsTest.class, ignoredTests); - suite.addTestSuite(TxDataConsistencyOnCommitFailureTest.class); + GridTestUtils.addTestIfNeeded(suite, TxDataConsistencyOnCommitFailureTest.class, ignoredTests); return suite; } From 339580feb9660af49d0575c23b7a71ab3dde85d1 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Wed, 5 Dec 2018 15:54:44 +0300 Subject: [PATCH 276/403] IGNITE-10049: MVCC: Create "Cache 4" test suite for MVCC mode. This closes #5499. --- ...ThroughDisabledTransactionalCacheTest.java | 8 + ...StoreSessionListenerLifecycleSelfTest.java | 8 + ...SessionListenerWriteBehindEnabledTest.java | 8 + ...CacheJdbcStoreSessionListenerSelfTest.java | 8 + ...ridCacheTxLoadFromStoreOnLockSelfTest.java | 8 + .../cache/CacheClientStoreSelfTest.java | 8 + .../cache/CacheConnectionLeakStoreTxTest.java | 20 + .../cache/CacheGetEntryAbstractTest.java | 4 +- ...EntryOptimisticReadCommittedSelfTest.java} | 2 +- ...ntryOptimisticRepeatableReadSelfTest.java} | 2 +- ...tEntryOptimisticSerializableSelfTest.java} | 2 +- ...ntryPessimisticReadCommittedSelfTest.java} | 2 +- ...ntryPessimisticRepeatableReadSelfTest.java | 105 ++++++ ...ntryPessimisticRepeatableReadSeltTest.java | 40 -- ...EntryPessimisticSerializableSelfTest.java} | 2 +- .../cache/CacheOffheapMapEntrySelfTest.java | 14 + .../CacheReadThroughLocalRestartSelfTest.java | 8 + .../CacheReadThroughRestartSelfTest.java | 11 + .../cache/CacheRemoveAllSelfTest.java | 9 + .../cache/CacheStopAndDestroySelfTest.java | 18 + ...StoreUsageMultinodeDynamicStartTxTest.java | 8 + ...eStoreUsageMultinodeStaticStartTxTest.java | 8 + .../CacheTxNotAllowReadFromBackupTest.java | 66 ++++ .../processors/cache/CrossCacheLockTest.java | 8 + .../cache/GridCacheAbstractSelfTest.java | 4 +- .../GridCacheMarshallingNodeJoinSelfTest.java | 8 + ...dCacheMultinodeUpdateAbstractSelfTest.java | 17 +- ...odeUpdateNearEnabledNoBackupsSelfTest.java | 6 +- ...cheMultinodeUpdateNearEnabledSelfTest.java | 14 +- ...dCacheStoreManagerDeserializationTest.java | 10 +- .../cache/GridCacheVersionMultinodeTest.java | 35 +- ...eCacheInvokeReadThroughSingleNodeTest.java | 46 +++ .../IgniteCacheInvokeReadThroughTest.java | 55 +++ .../IgniteCacheReadThroughStoreCallTest.java | 8 + .../IgniteCacheTxCopyOnReadDisabledTest.java | 7 + .../IgniteCacheTxLocalPeekModesTest.java | 8 + .../IgniteCacheTxLocalStoreValueTest.java | 8 + ...gniteCacheTxNearEnabledStoreValueTest.java | 8 + .../cache/IgniteCacheTxNearPeekModesTest.java | 9 + .../cache/IgniteCacheTxPeekModesTest.java | 9 + .../cache/IgniteCacheTxStoreValueTest.java | 8 + ...niteClientCacheInitializationFailTest.java | 37 +- .../cache/IgniteDynamicCacheFilterTest.java | 2 + ...amicCacheStartCoordinatorFailoverTest.java | 3 + ...ynamicCacheStartNoExchangeTimeoutTest.java | 36 +- .../IgniteDynamicCacheStartSelfTest.java | 14 + ...teDynamicCacheWithConfigStartSelfTest.java | 2 + ...IgniteDynamicClientCacheStartSelfTest.java | 8 +- ...IgniteStartCacheInTransactionSelfTest.java | 3 + .../distributed/CacheAffinityEarlyTest.java | 2 +- .../distributed/CacheGroupsPreloadTest.java | 44 +++ .../distributed/CacheStartOnJoinTest.java | 14 +- ...IgniteCacheCreatePutMultiNodeSelfTest.java | 17 +- .../distributed/IgniteCacheCreatePutTest.java | 11 + .../IgniteCacheFailedUpdateResponseTest.java | 41 +- .../IgniteCacheReadFromBackupTest.java | 89 ++++- .../IgniteCacheSingleGetMessageTest.java | 33 +- .../IgniteTxPreloadAbstractTest.java | 10 +- .../dht/IgniteCacheLockFailoverSelfTest.java | 8 + .../dht/IgniteCacheMultiTxLockSelfTest.java | 24 +- .../dht/IgniteCrossCacheMvccTxSelfTest.java | 39 ++ .../IgniteCrossCacheTxAbstractSelfTest.java | 181 +++++++++ .../dht/IgniteCrossCacheTxSelfTest.java | 155 +------- .../near/GridCacheNearTxPreloadSelfTest.java | 8 + .../GridCachePartitionedEventSelfTest.java | 8 + .../near/IgniteCacheNearOnlyTxTest.java | 8 + .../IgniteCacheNearReadCommittedTest.java | 8 + .../IgniteCacheLoadAllAbstractTest.java | 10 +- .../IgniteCacheTxLoaderWriterTest.java | 8 + .../IgniteCacheTxLocalLoadAllTest.java | 7 + ...teCacheTxLocalNoLoadPreviousValueTest.java | 8 + .../IgniteCacheTxLocalNoReadThroughTest.java | 8 + .../IgniteCacheTxLocalNoWriteThroughTest.java | 8 + ...eTxNearEnabledNoLoadPreviousValueTest.java | 8 + ...eCacheTxNearEnabledNoWriteThroughTest.java | 8 + .../IgniteCacheTxNoLoadPreviousValueTest.java | 8 + .../IgniteCacheTxNoReadThroughTest.java | 8 + .../IgniteCacheTxNoWriteThroughTest.java | 8 + .../IgniteCacheTxStoreSessionTest.java | 8 + ...StoreSessionWriteBehindCoalescingTest.java | 8 + ...iteCacheTxStoreSessionWriteBehindTest.java | 8 + ...sionedEntryLocalTransactionalSelfTest.java | 9 + .../testframework/MvccFeatureChecker.java | 20 + .../testsuites/IgniteCacheMvccTestSuite2.java | 1 + .../testsuites/IgniteCacheMvccTestSuite4.java | 197 ++++++++++ .../testsuites/IgniteCacheTestSuite4.java | 352 +++++++++--------- 86 files changed, 1729 insertions(+), 405 deletions(-) rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/{CacheGetEntryOptimisticReadCommittedSeltTest.java => CacheGetEntryOptimisticReadCommittedSelfTest.java} (95%) rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/{CacheGetEntryOptimisticRepeatableReadSeltTest.java => CacheGetEntryOptimisticRepeatableReadSelfTest.java} (95%) rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/{CacheGetEntryOptimisticSerializableSeltTest.java => CacheGetEntryOptimisticSerializableSelfTest.java} (95%) rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/{CacheGetEntryPessimisticReadCommittedSeltTest.java => CacheGetEntryPessimisticReadCommittedSelfTest.java} (95%) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/{CacheGetEntryPessimisticSerializableSeltTest.java => CacheGetEntryPessimisticSerializableSelfTest.java} (95%) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheMvccTxSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxAbstractSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite4.java diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java index 45038154ad60c..faf30b7c9474c 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreListenerRWThroughDisabledTransactionalCacheTest.java @@ -20,6 +20,7 @@ import java.util.Random; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -36,6 +37,13 @@ * and {@link CacheStoreSessionListener#onSessionEnd(CacheStoreSession, boolean)} are not executed. */ public class CacheStoreListenerRWThroughDisabledTransactionalCacheTest extends CacheStoreSessionListenerReadWriteThroughDisabledAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerLifecycleSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerLifecycleSelfTest.java index ff176c5f0f7c1..9bc3cdc759662 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerLifecycleSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerLifecycleSelfTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -49,6 +50,13 @@ public class CacheStoreSessionListenerLifecycleSelfTest extends GridCommonAbstra /** */ private static final Queue evts = new ConcurrentLinkedDeque<>(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java index c9a912ad51d57..3edd6b2c776e7 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/CacheStoreSessionListenerWriteBehindEnabledTest.java @@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore; import org.apache.ignite.resources.CacheStoreSessionResource; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * This class tests that calls of {@link CacheStoreSessionListener#onSessionStart(CacheStoreSession)} @@ -66,6 +67,13 @@ public class CacheStoreSessionListenerWriteBehindEnabledTest extends GridCacheAb /** */ private static final AtomicInteger uninitializedListenerCnt = new AtomicInteger(); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreSessionListenerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreSessionListenerSelfTest.java index 237cfeb363799..968dc0837da34 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreSessionListenerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcStoreSessionListenerSelfTest.java @@ -32,12 +32,20 @@ import org.apache.ignite.cache.store.CacheStoreSessionListenerAbstractSelfTest; import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.resources.CacheStoreSessionResource; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.h2.jdbcx.JdbcConnectionPool; /** * Tests for {@link CacheJdbcStoreSessionListener}. */ public class CacheJdbcStoreSessionListenerSelfTest extends CacheStoreSessionListenerAbstractSelfTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected Factory> storeFactory() { return new Factory>() { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/GridCacheTxLoadFromStoreOnLockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/GridCacheTxLoadFromStoreOnLockSelfTest.java index 6293723f18ae6..a1f18a7dd4a44 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/GridCacheTxLoadFromStoreOnLockSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/GridCacheTxLoadFromStoreOnLockSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -45,6 +46,13 @@ public class GridCacheTxLoadFromStoreOnLockSelfTest extends GridCommonAbstractTe /** */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheClientStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheClientStoreSelfTest.java index 87037918306e0..ecf9d2e5e9046 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheClientStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheClientStoreSelfTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; @@ -66,6 +67,13 @@ public class CacheClientStoreSelfTest extends GridCommonAbstractTest { /** */ private static volatile boolean loadedFromClient; + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java index 27dbe62577a61..db08d53371ae7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheConnectionLeakStoreTxTest.java @@ -169,6 +169,26 @@ public void testConnectionLeakOneBackupPessimisticReadCommittedLoadFromStore() t checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL, PESSIMISTIC, READ_COMMITTED); } + /** + * @throws Exception If failed. + */ + public void testConnectionLeakOneBackupMvccPessimisticRepeatableRead() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, PESSIMISTIC, REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testConnectionLeakOneBackupMvccPessimisticRepeatableReadLoadFromStore() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + isLoadFromStore = true; + + checkConnectionLeak(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, PESSIMISTIC, REPEATABLE_READ); + } + /** * @param atomicityMode Atomicity mode. * @param txConcurrency Transaction concurrency. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryAbstractTest.java index 6a7cfc6da4772..1a031c7f6fb79 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryAbstractTest.java @@ -171,8 +171,6 @@ public void testLocal() throws Exception { * @throws Exception If failed. */ public void testLocalTransactional() throws Exception { - // TODO: fails since d13520e9a05bd9e9b987529472d6317951b72f96, need to review changes. - CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); cfg.setWriteSynchronizationMode(FULL_SYNC); @@ -215,7 +213,7 @@ public void testReplicatedTransactional() throws Exception { * @param cfg Cache configuration. * @throws Exception If failed. */ - private void test(CacheConfiguration cfg) throws Exception { + protected void test(CacheConfiguration cfg) throws Exception { test(cfg, true); test(cfg, false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticReadCommittedSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticReadCommittedSelfTest.java similarity index 95% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticReadCommittedSeltTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticReadCommittedSelfTest.java index c04612d08d235..09275468b338e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticReadCommittedSeltTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticReadCommittedSelfTest.java @@ -23,7 +23,7 @@ /** * Test getEntry and getEntries methods. */ -public class CacheGetEntryOptimisticReadCommittedSeltTest extends CacheGetEntryAbstractTest { +public class CacheGetEntryOptimisticReadCommittedSelfTest extends CacheGetEntryAbstractTest { /** {@inheritDoc} */ @Override protected TransactionConcurrency concurrency() { return TransactionConcurrency.OPTIMISTIC; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticRepeatableReadSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticRepeatableReadSelfTest.java similarity index 95% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticRepeatableReadSeltTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticRepeatableReadSelfTest.java index 6153869821a0b..2c6a20440e196 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticRepeatableReadSeltTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticRepeatableReadSelfTest.java @@ -23,7 +23,7 @@ /** * Test getEntry and getEntries methods. */ -public class CacheGetEntryOptimisticRepeatableReadSeltTest extends CacheGetEntryAbstractTest { +public class CacheGetEntryOptimisticRepeatableReadSelfTest extends CacheGetEntryAbstractTest { /** {@inheritDoc} */ @Override protected TransactionConcurrency concurrency() { return TransactionConcurrency.OPTIMISTIC; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticSerializableSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticSerializableSelfTest.java similarity index 95% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticSerializableSeltTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticSerializableSelfTest.java index 6ded4a9acd2e2..63161e245d401 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticSerializableSeltTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryOptimisticSerializableSelfTest.java @@ -23,7 +23,7 @@ /** * Test getEntry and getEntries methods. */ -public class CacheGetEntryOptimisticSerializableSeltTest extends CacheGetEntryAbstractTest { +public class CacheGetEntryOptimisticSerializableSelfTest extends CacheGetEntryAbstractTest { /** {@inheritDoc} */ @Override protected TransactionConcurrency concurrency() { return TransactionConcurrency.OPTIMISTIC; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticReadCommittedSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticReadCommittedSelfTest.java similarity index 95% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticReadCommittedSeltTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticReadCommittedSelfTest.java index 975d2718af5e5..0a291f768d997 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticReadCommittedSeltTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticReadCommittedSelfTest.java @@ -23,7 +23,7 @@ /** * Test getEntry and getEntries methods. */ -public class CacheGetEntryPessimisticReadCommittedSeltTest extends CacheGetEntryAbstractTest { +public class CacheGetEntryPessimisticReadCommittedSelfTest extends CacheGetEntryAbstractTest { /** {@inheritDoc} */ @Override protected TransactionConcurrency concurrency() { return TransactionConcurrency.PESSIMISTIC; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSelfTest.java new file mode 100644 index 0000000000000..906b726a296ca --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSelfTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.LOCAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheMode.REPLICATED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * Test getEntry and getEntries methods. + */ +public class CacheGetEntryPessimisticRepeatableReadSelfTest extends CacheGetEntryAbstractTest { + /** {@inheritDoc} */ + @Override protected TransactionConcurrency concurrency() { + return TransactionConcurrency.PESSIMISTIC; + } + + /** {@inheritDoc} */ + @Override protected TransactionIsolation isolation() { + return TransactionIsolation.REPEATABLE_READ; + } + + /** + * @throws Exception If failed. + */ + public void testNearTransactionalMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + cfg.setWriteSynchronizationMode(FULL_SYNC); + cfg.setCacheMode(PARTITIONED); + cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + cfg.setName("nearT"); + cfg.setNearConfiguration(new NearCacheConfiguration()); + + test(cfg); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionedTransactionalMvcc() throws Exception { + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + cfg.setWriteSynchronizationMode(FULL_SYNC); + cfg.setCacheMode(PARTITIONED); + cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + cfg.setName("partitionedT"); + + test(cfg); + } + + /** + * @throws Exception If failed. + */ + public void testLocalTransactionalMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + cfg.setWriteSynchronizationMode(FULL_SYNC); + cfg.setCacheMode(LOCAL); + cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + cfg.setName("localT"); + + test(cfg); + } + + /** + * @throws Exception If failed. + */ + public void testReplicatedTransactionalMvcc() throws Exception { + CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + cfg.setWriteSynchronizationMode(FULL_SYNC); + cfg.setCacheMode(REPLICATED); + cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + cfg.setName("replicatedT"); + + test(cfg); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java deleted file mode 100644 index 4aa693c29c1b3..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import org.apache.ignite.transactions.TransactionConcurrency; -import org.apache.ignite.transactions.TransactionIsolation; - -/** - * Test getEntry and getEntries methods. - */ -public class CacheGetEntryPessimisticRepeatableReadSeltTest extends CacheGetEntryAbstractTest { - /** {@inheritDoc} */ - @Override protected TransactionConcurrency concurrency() { - return TransactionConcurrency.PESSIMISTIC; - } - - /** {@inheritDoc} */ - @Override protected TransactionIsolation isolation() { - return TransactionIsolation.REPEATABLE_READ; - } - - @Override public void testReplicatedTransactional() throws Exception { - super.testReplicatedTransactional(); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticSerializableSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticSerializableSelfTest.java similarity index 95% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticSerializableSeltTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticSerializableSelfTest.java index 70f71ced31342..dfaed7e4149fa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticSerializableSeltTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticSerializableSelfTest.java @@ -23,7 +23,7 @@ /** * Test getEntry and getEntries methods. */ -public class CacheGetEntryPessimisticSerializableSeltTest extends CacheGetEntryAbstractTest { +public class CacheGetEntryPessimisticSerializableSelfTest extends CacheGetEntryAbstractTest { /** {@inheritDoc} */ @Override protected TransactionConcurrency concurrency() { return TransactionConcurrency.PESSIMISTIC; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheOffheapMapEntrySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheOffheapMapEntrySelfTest.java index e520d455c1aaa..f2e12adc884a4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheOffheapMapEntrySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheOffheapMapEntrySelfTest.java @@ -25,9 +25,11 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry; import org.apache.ignite.internal.processors.cache.local.GridLocalCacheEntry; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -76,6 +78,9 @@ private CacheConfiguration cacheConfiguration(String gridName, cfg.setAtomicityMode(atomicityMode); cfg.setName(cacheName); + if (atomicityMode == TRANSACTIONAL_SNAPSHOT && !MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) + cfg.setNearConfiguration(null); + return cfg; } @@ -87,13 +92,22 @@ public void testCacheMapEntry() throws Exception { checkCacheMapEntry(TRANSACTIONAL, LOCAL, GridLocalCacheEntry.class); + if (MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.LOCAL_CACHE)) + checkCacheMapEntry(TRANSACTIONAL_SNAPSHOT, LOCAL, GridLocalCacheEntry.class); + checkCacheMapEntry(ATOMIC, PARTITIONED, GridNearCacheEntry.class); checkCacheMapEntry(TRANSACTIONAL, PARTITIONED, GridNearCacheEntry.class); + if (MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + checkCacheMapEntry(TRANSACTIONAL_SNAPSHOT, PARTITIONED, GridDhtCacheEntry.class); + checkCacheMapEntry(ATOMIC, REPLICATED, GridDhtCacheEntry.class); checkCacheMapEntry(TRANSACTIONAL, REPLICATED, GridDhtCacheEntry.class); + + if (MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + checkCacheMapEntry(TRANSACTIONAL_SNAPSHOT, REPLICATED, GridDhtCacheEntry.class); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughLocalRestartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughLocalRestartSelfTest.java index 58fa8d6f62828..5e8b1eb442abe 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughLocalRestartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughLocalRestartSelfTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -29,4 +30,11 @@ public class CacheReadThroughLocalRestartSelfTest extends CacheReadThroughRestar @Override protected CacheMode cacheMode() { return LOCAL; } + + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughRestartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughRestartSelfTest.java index 422ed586be455..5b79f7febb704 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughRestartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheReadThroughRestartSelfTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -41,6 +42,13 @@ public class CacheReadThroughRestartSelfTest extends GridCacheAbstractSelfTest { /** */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 2; @@ -116,6 +124,9 @@ private void testReadThroughInTx(boolean needVer) throws Exception { for (TransactionConcurrency txConcurrency : TransactionConcurrency.values()) { for (TransactionIsolation txIsolation : TransactionIsolation.values()) { + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(txConcurrency, txIsolation)) + continue; + try (Transaction tx = ignite.transactions().txStart(txConcurrency, txIsolation, 100000, 1000)) { for (int k = 0; k < 1000; k++) { String key = "key" + k; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRemoveAllSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRemoveAllSelfTest.java index a27bdda62a177..cccb92a82636b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRemoveAllSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRemoveAllSelfTest.java @@ -24,11 +24,20 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Test remove all method. */ public class CacheRemoveAllSelfTest extends GridCacheAbstractSelfTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10082"); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected long getTestTimeout() { return 2 * 60 * 1000; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java index 6239b525d9eb4..a204ddf0920f0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -47,6 +48,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -150,6 +152,7 @@ private CacheConfiguration getDhtConfig() { cfg.setName(CACHE_NAME_DHT); cfg.setCacheMode(PARTITIONED); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); cfg.setNearConfiguration(null); return cfg; @@ -163,6 +166,7 @@ private CacheConfiguration getClientConfig() { cfg.setName(CACHE_NAME_CLIENT); cfg.setCacheMode(PARTITIONED); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); cfg.setNearConfiguration(null); return cfg; @@ -176,6 +180,7 @@ private CacheConfiguration getNearConfig() { cfg.setName(CACHE_NAME_NEAR); cfg.setCacheMode(PARTITIONED); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); cfg.setNearConfiguration(new NearCacheConfiguration()); return cfg; @@ -190,6 +195,7 @@ private CacheConfiguration getLocalConfig() { cfg.setName(CACHE_NAME_LOC); cfg.setCacheMode(LOCAL); cfg.setNearConfiguration(null); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); return cfg; } @@ -280,6 +286,8 @@ private void clientDestroy() throws Exception { * @throws Exception If failed. */ public void testNearDoubleDestroy() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + startGridsMultiThreaded(gridCount()); nearDestroy(); @@ -320,6 +328,8 @@ private void nearDestroy() throws Exception { * @throws Exception If failed. */ public void testLocalDoubleDestroy() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + startGridsMultiThreaded(gridCount()); localDestroy(); @@ -563,6 +573,8 @@ public void testClientCloseWithTry() throws Exception { * @throws Exception If failed. */ public void testNearClose() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + startGridsMultiThreaded(gridCount()); IgniteCache cache0 = grid(0).getOrCreateCache(getNearConfig()); @@ -636,6 +648,8 @@ public void testNearClose() throws Exception { * @throws Exception If failed. */ public void testNearCloseWithTry() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + startGridsMultiThreaded(gridCount()); String curVal = null; @@ -674,6 +688,8 @@ public void testNearCloseWithTry() throws Exception { * @throws Exception If failed. */ public void testLocalClose() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + memCfg = new DataStorageConfiguration(); startGridsMultiThreaded(gridCount()); @@ -725,6 +741,8 @@ public void testLocalClose() throws Exception { * @throws Exception If failed. */ public void testLocalCloseWithTry() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + memCfg = new DataStorageConfiguration(); startGridsMultiThreaded(gridCount()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeDynamicStartTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeDynamicStartTxTest.java index 4511fc5a3d826..e217a1f878382 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeDynamicStartTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeDynamicStartTxTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -25,6 +26,13 @@ * */ public class CacheStoreUsageMultinodeDynamicStartTxTest extends CacheStoreUsageMultinodeDynamicStartAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeStaticStartTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeStaticStartTxTest.java index 2f11fc862b47c..8fa7e65b2c37e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeStaticStartTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStoreUsageMultinodeStaticStartTxTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -25,6 +26,13 @@ * */ public class CacheStoreUsageMultinodeStaticStartTxTest extends CacheStoreUsageMultinodeStaticStartAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxNotAllowReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxNotAllowReadFromBackupTest.java index f0ec084a55764..fad2c615e701e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxNotAllowReadFromBackupTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxNotAllowReadFromBackupTest.java @@ -188,6 +188,72 @@ public void testBackupConsistencyPartitionedFullSync() throws Exception { checkBackupConsistencyGetAll(cfg, TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE); } + /** + * @throws Exception If failed. + */ + public void testBackupConsistencyReplicatedMvcc() throws Exception { + CacheConfiguration cfg = new CacheConfiguration<>("test-cache"); + + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC); + cfg.setCacheMode(CacheMode.REPLICATED); + cfg.setReadFromBackup(false); + + checkBackupConsistency(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + + checkBackupConsistencyGetAll(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testBackupConsistencyReplicatedFullSyncMvcc() throws Exception { + CacheConfiguration cfg = new CacheConfiguration<>("test-cache"); + + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cfg.setCacheMode(CacheMode.REPLICATED); + cfg.setReadFromBackup(false); + + checkBackupConsistency(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + + checkBackupConsistencyGetAll(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testBackupConsistencyPartitionedMvcc() throws Exception { + CacheConfiguration cfg = new CacheConfiguration<>("test-cache"); + + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC); + cfg.setCacheMode(CacheMode.PARTITIONED); + cfg.setBackups(NODES - 1); + cfg.setReadFromBackup(false); + + checkBackupConsistency(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + + checkBackupConsistencyGetAll(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + } + + /** + * @throws Exception If failed. + */ + public void testBackupConsistencyPartitionedFullSyncMvcc() throws Exception { + CacheConfiguration cfg = new CacheConfiguration<>("test-cache"); + + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cfg.setCacheMode(CacheMode.PARTITIONED); + cfg.setBackups(NODES - 1); + cfg.setReadFromBackup(false); + + checkBackupConsistency(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + + checkBackupConsistencyGetAll(cfg, TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ); + } + /** * @param ccfg Cache configuration. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheLockTest.java index 8f0e20f604ece..d609ec7ca7134 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheLockTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheLockTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -45,6 +46,13 @@ public class CrossCacheLockTest extends GridCommonAbstractTest { /** */ private static final String CACHE2 = "cache2"; + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java index 68104bfa721b2..9504b812ee83d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java @@ -53,6 +53,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.Nullable; @@ -92,7 +93,8 @@ public abstract class GridCacheAbstractSelfTest extends GridCommonAbstractTest { assert cnt >= 1 : "At least one grid must be started"; - initStoreStrategy(); + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.CACHE_STORE)) + initStoreStrategy(); startGrids(cnt); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMarshallingNodeJoinSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMarshallingNodeJoinSelfTest.java index df3430f2bbfc2..c837e2132cd5b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMarshallingNodeJoinSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMarshallingNodeJoinSelfTest.java @@ -39,6 +39,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -51,6 +52,13 @@ public class GridCacheMarshallingNodeJoinSelfTest extends GridCommonAbstractTest /** */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateAbstractSelfTest.java index 800f4bac7b67c..2027117b6be38 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateAbstractSelfTest.java @@ -29,6 +29,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.testframework.MvccFeatureChecker.assertMvccWriteConflict; /** * Multinode update test. @@ -97,8 +98,20 @@ public void testInvoke() throws Exception { final IgniteCache cache = grid(idx).cache(DEFAULT_CACHE_NAME); - for (int i = 0; i < ITERATIONS_PER_THREAD && !failed; i++) - cache.invoke(key, new IncProcessor()); + for (int i = 0; i < ITERATIONS_PER_THREAD && !failed; i++) { + boolean updated = false; + + while (!updated) { + try { + cache.invoke(key, new IncProcessor()); + + updated = true; + } + catch (Exception e) { + assertMvccWriteConflict(e); + } + } + } return null; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest.java index fbfb994fc9f21..aa9f0293868d6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * @@ -39,6 +40,9 @@ public class GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest extends GridCa /** {@inheritDoc} */ @Override public void testInvoke() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-809"); + if (!MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-809"); + else + super.testInvoke(); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledSelfTest.java index 44f2811638910..96f3927b9506b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMultinodeUpdateNearEnabledSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -27,7 +28,13 @@ */ public class GridCacheMultinodeUpdateNearEnabledSelfTest extends GridCacheMultinodeUpdateAbstractSelfTest { /** {@inheritDoc} */ - @SuppressWarnings("RedundantMethodOverride") + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + + /** {@inheritDoc} */ @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); } @@ -40,6 +47,9 @@ public class GridCacheMultinodeUpdateNearEnabledSelfTest extends GridCacheMultin /** {@inheritDoc} */ @Override public void testInvoke() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-809"); + if (!MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-809"); + else + super.testInvoke(); } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStoreManagerDeserializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStoreManagerDeserializationTest.java index a1623d2880497..d59a6912fa6be 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStoreManagerDeserializationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheStoreManagerDeserializationTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -66,6 +67,13 @@ public class GridCacheStoreManagerDeserializationTest extends GridCommonAbstract /** Test cache name. */ protected static final String CACHE_NAME = "cache_name"; + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** * @return Cache mode. */ @@ -124,7 +132,7 @@ protected CacheConfiguration cacheConfiguration() { cc.setBackups(0); - cc.setAtomicityMode(CacheAtomicityMode.ATOMIC); + cc.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); return cc; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionMultinodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionMultinodeTest.java index 0fd3fb9ca5ca3..ccecfb787f76e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionMultinodeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheVersionMultinodeTest.java @@ -33,6 +33,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; @@ -106,6 +107,30 @@ public void testVersionTxNearEnabled() throws Exception { checkVersion(); } + /** + * @throws Exception If failed. + */ + public void testVersionMvccTx() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + checkVersion(); + } + + /** + * @throws Exception If failed. + */ + public void testVersionMvccTxNearEnabled() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + near = true; + + checkVersion(); + } + /** * @throws Exception If failed. */ @@ -138,17 +163,19 @@ private void checkVersion() throws Exception { checkVersion(String.valueOf(i), null); // Update. } - if (atomicityMode == TRANSACTIONAL) { + if (atomicityMode != ATOMIC) { for (int i = 100; i < 200; i++) { checkVersion(String.valueOf(i), PESSIMISTIC); // Create. checkVersion(String.valueOf(i), PESSIMISTIC); // Update. } - for (int i = 200; i < 300; i++) { - checkVersion(String.valueOf(i), OPTIMISTIC); // Create. + if (atomicityMode != TRANSACTIONAL_SNAPSHOT) { + for (int i = 200; i < 300; i++) { + checkVersion(String.valueOf(i), OPTIMISTIC); // Create. - checkVersion(String.valueOf(i), OPTIMISTIC); // Update. + checkVersion(String.valueOf(i), OPTIMISTIC); // Update. + } } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughSingleNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughSingleNodeTest.java index 406e5afe9ff7d..cb4410a58bfe6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughSingleNodeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughSingleNodeTest.java @@ -17,8 +17,11 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.testframework.MvccFeatureChecker; + import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -27,6 +30,13 @@ * */ public class IgniteCacheInvokeReadThroughSingleNodeTest extends IgniteCacheInvokeReadThroughAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected void startNodes() throws Exception { startGrid(0); @@ -87,4 +97,40 @@ public void testInvokeReadThroughTxReplicated() throws Exception { public void testInvokeReadThroughTxLocal() throws Exception { invokeReadThrough(cacheConfiguration(LOCAL, TRANSACTIONAL, 0, false)); } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTx() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, false)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTxNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, true)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(REPLICATED, TRANSACTIONAL_SNAPSHOT, 0, false)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(LOCAL, TRANSACTIONAL_SNAPSHOT, 0, false)); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughTest.java index 8fd37588a43eb..3866a1865cf07 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeReadThroughTest.java @@ -17,8 +17,11 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.testframework.MvccFeatureChecker; + import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -26,6 +29,13 @@ * */ public class IgniteCacheInvokeReadThroughTest extends IgniteCacheInvokeReadThroughAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected void startNodes() throws Exception { startGridsMultiThreaded(4); @@ -104,4 +114,49 @@ public void testInvokeReadThroughTxNearCache() throws Exception { public void testInvokeReadThroughTxReplicated() throws Exception { invokeReadThrough(cacheConfiguration(REPLICATED, TRANSACTIONAL, 0, false)); } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTx0() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 0, false)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTx1() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, false)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTx2() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 2, false)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTxNearCache() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, true)); + } + + /** + * @throws Exception If failed. + */ + public void testInvokeReadThroughMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + + invokeReadThrough(cacheConfiguration(REPLICATED, TRANSACTIONAL_SNAPSHOT, 0, false)); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughStoreCallTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughStoreCallTest.java index 423c4a156575e..834be7f798e0e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughStoreCallTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheReadThroughStoreCallTest.java @@ -39,6 +39,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -64,6 +65,13 @@ public class IgniteCacheReadThroughStoreCallTest extends GridCommonAbstractTest /** */ protected boolean client; + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxCopyOnReadDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxCopyOnReadDisabledTest.java index 2909f0a296cf8..27bf171e20be7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxCopyOnReadDisabledTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxCopyOnReadDisabledTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -25,6 +26,12 @@ * */ public class IgniteCacheTxCopyOnReadDisabledTest extends IgniteCacheCopyOnReadDisabledAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalPeekModesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalPeekModesTest.java index 3439590040fcf..26dbb7e129a44 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalPeekModesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalPeekModesTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -27,6 +28,13 @@ * */ public class IgniteCacheTxLocalPeekModesTest extends IgniteCachePeekModesAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalStoreValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalStoreValueTest.java index b3987265c1da4..a25fd90c00b31 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalStoreValueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxLocalStoreValueTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxLocalStoreValueTest extends IgniteCacheStoreValueAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearEnabledStoreValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearEnabledStoreValueTest.java index fc719fa9862c2..4dc2cf3baa80f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearEnabledStoreValueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearEnabledStoreValueTest.java @@ -18,11 +18,19 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * */ public class IgniteCacheTxNearEnabledStoreValueTest extends IgniteCacheTxStoreValueTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearPeekModesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearPeekModesTest.java index aa4faaf6049ea..bd039831acb69 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearPeekModesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxNearPeekModesTest.java @@ -17,10 +17,19 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.testframework.MvccFeatureChecker; + /** * Tests peek modes with near tx cache. */ public class IgniteCacheTxNearPeekModesTest extends IgniteCacheTxPeekModesTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected boolean hasNearCache() { return true; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java index c6c937bd4698f..b43ab9ad67266 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -41,4 +42,12 @@ public class IgniteCacheTxPeekModesTest extends IgniteCachePeekModesAbstractTest @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; } + + /** {@inheritDoc} */ + @Override public void testLocalEntries() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10167"); + + super.testLocalEntries(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxStoreValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxStoreValueTest.java index f3f52ebdf196d..f137e523e868a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxStoreValueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxStoreValueTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxStoreValueTest extends IgniteCacheStoreValueAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.INTERCEPTOR); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 4; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java index cea591082d7f5..41227ca523f1f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java @@ -81,12 +81,18 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT /** Tx cache name. */ private static final String TX_CACHE_NAME = "tx-cache"; + /** Mvcc tx cache name. */ + private static final String MVCC_TX_CACHE_NAME = "mvcc-tx-cache"; + /** Near atomic cache name. */ private static final String NEAR_ATOMIC_CACHE_NAME = "near-atomic-cache"; /** Near tx cache name. */ private static final String NEAR_TX_CACHE_NAME = "near-tx-cache"; + /** Near mvcc tx cache name. */ + private static final String NEAR_MVCC_TX_CACHE_NAME = "near-mvcc-tx-cache"; + /** Failed caches. */ private static final Set FAILED_CACHES; @@ -97,6 +103,8 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT set.add(TX_CACHE_NAME); set.add(NEAR_ATOMIC_CACHE_NAME); set.add(NEAR_TX_CACHE_NAME); + set.add(MVCC_TX_CACHE_NAME); + set.add(NEAR_MVCC_TX_CACHE_NAME); FAILED_CACHES = Collections.unmodifiableSet(set); } @@ -124,7 +132,13 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT ccfg2.setName(TX_CACHE_NAME); ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - cfg.setCacheConfiguration(ccfg1, ccfg2); + CacheConfiguration ccfg3 = new CacheConfiguration<>(); + + ccfg3.setIndexedTypes(Integer.class, String.class); + ccfg3.setName(MVCC_TX_CACHE_NAME); + ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + + cfg.setCacheConfiguration(ccfg1, ccfg2, ccfg3); } else { GridQueryProcessor.idxCls = FailedIndexing.class; @@ -149,6 +163,13 @@ public void testTransactionalCacheInitialization() throws Exception { checkCacheInitialization(TX_CACHE_NAME); } + /** + * @throws Exception If failed. + */ + public void testMvccTransactionalCacheInitialization() throws Exception { + checkCacheInitialization(MVCC_TX_CACHE_NAME); + } + /** * @throws Exception If failed. */ @@ -163,6 +184,15 @@ public void testTransactionalNearCacheInitialization() throws Exception { checkCacheInitialization(NEAR_TX_CACHE_NAME); } + /** + * @throws Exception If failed. + */ + public void testMvccTransactionalNearCacheInitialization() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + checkCacheInitialization(NEAR_MVCC_TX_CACHE_NAME); + } + /** * @param cacheName Cache name. * @throws Exception If failed. @@ -201,12 +231,15 @@ private void checkFailedCache(final Ignite client, final String cacheName) { IgniteCache cache; // Start cache with near enabled. - if (NEAR_ATOMIC_CACHE_NAME.equals(cacheName) || NEAR_TX_CACHE_NAME.equals(cacheName)) { + if (NEAR_ATOMIC_CACHE_NAME.equals(cacheName) || NEAR_TX_CACHE_NAME.equals(cacheName) || + NEAR_MVCC_TX_CACHE_NAME.equals(cacheName)) { CacheConfiguration ccfg = new CacheConfiguration(cacheName) .setNearConfiguration(new NearCacheConfiguration()); if (NEAR_TX_CACHE_NAME.equals(cacheName)) ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + else if (NEAR_MVCC_TX_CACHE_NAME.equals(cacheName)) + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); cache = client.getOrCreateCache(ccfg); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheFilterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheFilterTest.java index 53c1cb29fe452..4d741728f37c3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheFilterTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheFilterTest.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -60,6 +61,7 @@ public class IgniteDynamicCacheFilterTest extends GridCommonAbstractTest { ccfg.setRebalanceMode(SYNC); ccfg.setNodeFilter(new TestNodeFilter("A")); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); if (attrVal != null) { Map attrs = new HashMap<>(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java index 36e1879a1c96b..38c0914479014 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.affinity.AffinityFunctionContext; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.ClusterNode; @@ -122,6 +123,8 @@ public void testCoordinatorFailure() throws Exception { cfg.setName("test-coordinator-failover"); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(2))); GridTestUtils.runAsync(new Callable() { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartNoExchangeTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartNoExchangeTimeoutTest.java index c3e3e884cbc5d..a00fd244a0836 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartNoExchangeTimeoutTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartNoExchangeTimeoutTest.java @@ -45,6 +45,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; /** @@ -400,7 +401,7 @@ private List cacheConfigurations() { { CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - ccfg.setName("cache-4"); + ccfg.setName("cache-6"); ccfg.setAtomicityMode(TRANSACTIONAL); ccfg.setBackups(1); ccfg.setWriteSynchronizationMode(FULL_SYNC); @@ -408,6 +409,39 @@ private List cacheConfigurations() { res.add(ccfg); } + { + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + ccfg.setName("cache-7"); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + ccfg.setBackups(0); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + res.add(ccfg); + } + + { + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + ccfg.setName("cache-8"); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + ccfg.setBackups(1); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + res.add(ccfg); + } + + { + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + ccfg.setName("cache-9"); + ccfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); + ccfg.setBackups(1); + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + res.add(ccfg); + } + return res; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java index 53de23f90a602..b5b83ad6e8e61 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartSelfTest.java @@ -278,6 +278,13 @@ public void testStartStopCacheSimpleTransactional() throws Exception { checkStartStopCacheSimple(CacheAtomicityMode.TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testStartStopCacheSimpleTransactionalMvcc() throws Exception { + checkStartStopCacheSimple(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ @@ -292,6 +299,13 @@ public void testStartStopCachesSimpleTransactional() throws Exception { checkStartStopCachesSimple(CacheAtomicityMode.TRANSACTIONAL); } + /** + * @throws Exception If failed. + */ + public void testStartStopCachesSimpleTransactionalMvcc() throws Exception { + checkStartStopCachesSimple(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheWithConfigStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheWithConfigStartSelfTest.java index ec6b82de7c6d0..d366306111a2e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheWithConfigStartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheWithConfigStartSelfTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; @@ -63,6 +64,7 @@ private CacheConfiguration cacheConfiguration() { CacheConfiguration ccfg = new CacheConfiguration<>(CACHE_NAME); ccfg.setIndexedTypes(String.class, String.class); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); return ccfg; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicClientCacheStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicClientCacheStartSelfTest.java index 0cb08561521b0..69bf23323f2ac 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicClientCacheStartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicClientCacheStartSelfTest.java @@ -44,7 +44,10 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; -import static org.apache.ignite.cache.CacheAtomicityMode.*; +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheAtomicityMode.values; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME; @@ -394,8 +397,9 @@ public void testStartNewAndClientCaches() throws Exception { cfgs.addAll(cacheConfigurations(null, ATOMIC)); cfgs.addAll(cacheConfigurations(null, TRANSACTIONAL)); + cfgs.addAll(cacheConfigurations(null, TRANSACTIONAL_SNAPSHOT)); - assertEquals(6, cfgs.size()); + assertEquals(9, cfgs.size()); Collection caches = client.getOrCreateCaches(cfgs); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteStartCacheInTransactionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteStartCacheInTransactionSelfTest.java index b037a7b8ef6a8..f4d3b253ac46a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteStartCacheInTransactionSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteStartCacheInTransactionSelfTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -250,6 +251,8 @@ public void testLockCache() throws Exception { if (atomicityMode() != TRANSACTIONAL) return; + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + final Ignite ignite = grid(0); final String key = "key"; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAffinityEarlyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAffinityEarlyTest.java index 46669acc56807..2f04870e7819a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAffinityEarlyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAffinityEarlyTest.java @@ -156,7 +156,7 @@ private IgniteCache getCache(Ignite grid) { CacheConfiguration ccfg = defaultCacheConfiguration(); ccfg.setCacheMode(CacheMode.PARTITIONED); - ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); ccfg.setBackups(1); ccfg.setNearConfiguration(null); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheGroupsPreloadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheGroupsPreloadTest.java index 88596380c2aca..af223f4560dd6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheGroupsPreloadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheGroupsPreloadTest.java @@ -103,6 +103,17 @@ public void testCachePreload2() throws Exception { cachePreloadTest(); } + /** + * @throws Exception If failed. + */ + public void testCachePreloadMvcc2() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + atomicityMode = CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + + cachePreloadTest(); + } + /** * @throws Exception If failed. */ @@ -122,6 +133,16 @@ public void testCachePreload4() throws Exception { cachePreloadTest(); } + /** + * @throws Exception If failed. + */ + public void testCachePreloadMvcc4() throws Exception { + cacheMode = CacheMode.REPLICATED; + atomicityMode = CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + + cachePreloadTest(); + } + /** * @throws Exception If failed. */ @@ -141,6 +162,18 @@ public void testCachePreload6() throws Exception { cachePreloadTest(); } + /** + * @throws Exception If failed. + */ + public void testCachePreloadMvcc6() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + + sameGrp = false; + atomicityMode = CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + + cachePreloadTest(); + } + /** * @throws Exception If failed. */ @@ -162,6 +195,17 @@ public void testCachePreload8() throws Exception { cachePreloadTest(); } + /** + * @throws Exception If failed. + */ + public void testCachePreloadMvcc8() throws Exception { + sameGrp = false; + cacheMode = CacheMode.REPLICATED; + atomicityMode = CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + + cachePreloadTest(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartOnJoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartOnJoinTest.java index d59f4709506d1..54889e4273ead 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartOnJoinTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartOnJoinTest.java @@ -46,6 +46,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.testframework.MvccFeatureChecker.assertMvccWriteConflict; /** * @@ -184,7 +185,18 @@ private void doTest(final boolean createCache) throws Exception { if (createCache) { for (int c = 0; c < 5; c++) { for (IgniteCache cache : node.getOrCreateCaches(cacheConfigurations())) { - cache.put(c, c); + boolean updated = false; + + while (!updated) { + try { + cache.put(c, c); + + updated = true; + } + catch (Exception e) { + assertMvccWriteConflict(e); + } + } assertEquals(c, cache.get(c)); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutMultiNodeSelfTest.java index 23fc941472bc1..0b021a8474b39 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutMultiNodeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutMultiNodeSelfTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -94,8 +95,18 @@ public void testStartNodes() throws Exception { IgniteCache cache = getCache(ignite, cacheName); - for (int i = 0; i < 100; i++) - cache.getAndPut(i, i); + for (int i = 0; i < 100; i++) { + while (true) { + try { + cache.getAndPut(i, i); + + break; + } + catch (Exception e) { + MvccFeatureChecker.assertMvccWriteConflict(e); + } + } + } barrier.await(); @@ -139,7 +150,7 @@ private IgniteCache getCache(Ignite grid, String cacheName) { CacheConfiguration ccfg = new CacheConfiguration<>(cacheName); ccfg.setCacheMode(CacheMode.PARTITIONED); - ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); ccfg.setBackups(1); ccfg.setNearConfiguration(null); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java index 646084c869a32..5d6e895868ac7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java @@ -37,9 +37,11 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; +import static org.apache.ignite.testframework.MvccFeatureChecker.assertMvccWriteConflict; /** * @@ -149,6 +151,7 @@ public void testUpdatesAndCacheStart() throws Exception { ignite0.createCache(cacheConfiguration("atomic-cache", ATOMIC)); ignite0.createCache(cacheConfiguration("tx-cache", TRANSACTIONAL)); + ignite0.createCache(cacheConfiguration("mvcc-tx-cache", TRANSACTIONAL_SNAPSHOT)); final long stopTime = System.currentTimeMillis() + 60_000; @@ -162,6 +165,7 @@ public void testUpdatesAndCacheStart() throws Exception { IgniteCache cache1 = node.cache("atomic-cache"); IgniteCache cache2 = node.cache("tx-cache"); + IgniteCache cache3 = node.cache("mvcc-tx-cache"); ThreadLocalRandom rnd = ThreadLocalRandom.current(); @@ -174,6 +178,13 @@ public void testUpdatesAndCacheStart() throws Exception { cache2.put(key, key); + try { + cache3.put(key, key); + } + catch (Exception e) { + assertMvccWriteConflict(e); // Do not retry. + } + if (iter++ % 1000 == 0) log.info("Update iteration: " + iter); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java index ebcff7c43898e..cf7489fce660d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheFailedUpdateResponseTest.java @@ -38,11 +38,13 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.testframework.GridTestUtils.assertThrows; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; @@ -51,7 +53,7 @@ import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; /** - * Checks that no future hangs on non-srializable exceptions and values. + * Checks that no future hangs on non-serializable exceptions and values. */ public class IgniteCacheFailedUpdateResponseTest extends GridCommonAbstractTest { /** Atomic cache. */ @@ -60,25 +62,34 @@ public class IgniteCacheFailedUpdateResponseTest extends GridCommonAbstractTest /** Tx cache. */ private static final String TX_CACHE = "tx"; + /** Mvcc tx cache. */ + private static final String MVCC_TX_CACHE = "mvcc-tx"; + /** Atomic cache. */ private IgniteCache atomicCache; /** Tx cache. */ private IgniteCache txCache; + /** Mvcc tx cache. */ + private IgniteCache mvccTxCache; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); CacheConfiguration atomicCfg = new CacheConfiguration(ATOMIC_CACHE); CacheConfiguration txCfg = new CacheConfiguration(TX_CACHE); + CacheConfiguration mvccTxCfg = new CacheConfiguration(MVCC_TX_CACHE); atomicCfg.setBackups(1); txCfg.setBackups(1); + mvccTxCfg.setBackups(1); txCfg.setAtomicityMode(TRANSACTIONAL); + mvccTxCfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT); - cfg.setCacheConfiguration(atomicCfg, txCfg); + cfg.setCacheConfiguration(atomicCfg, txCfg, mvccTxCfg); cfg.setClientMode(igniteInstanceName.contains("client")); @@ -98,6 +109,7 @@ public class IgniteCacheFailedUpdateResponseTest extends GridCommonAbstractTest @Override protected void beforeTest() throws Exception { atomicCache = grid("client").cache(ATOMIC_CACHE); txCache = grid("client").cache(TX_CACHE); + mvccTxCache = grid("client").cache(MVCC_TX_CACHE); } /** @@ -134,11 +146,32 @@ public void testInvokeTx() throws Exception { doInTransaction(client, OPTIMISTIC, SERIALIZABLE, clos); } + /** + * @throws Exception If failed. + */ + public void testInvokeMvccTx() throws Exception { + testInvoke(mvccTxCache); + testInvokeAll(mvccTxCache); + + IgniteEx client = grid("client"); + + Callable clos = new Callable() { + @Override public Object call() throws Exception { + testInvoke(mvccTxCache); + testInvokeAll(mvccTxCache); + + return null; + } + }; + + doInTransaction(client, PESSIMISTIC, REPEATABLE_READ, clos); + } + /** * @param cache Cache. */ private void testInvoke(final IgniteCache cache) throws Exception { - Class exp = grid("client").transactions().tx() == null + Class exp = grid("client").transactions().tx() == null || ((IgniteCacheProxy)cache).context().mvccEnabled() ? EntryProcessorException.class : NonSerializableException.class; @@ -174,7 +207,7 @@ private void testInvokeAll(final IgniteCache cache) throws Excep assertNotNull(epRes); // In transactions EP will be invoked locally. - Class exp = grid("client").transactions().tx() == null + Class exp = grid("client").transactions().tx() == null || ((IgniteCacheProxy)cache).context().mvccEnabled() ? EntryProcessorException.class : NonSerializableException.class; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheReadFromBackupTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheReadFromBackupTest.java index 2bb5fbba81e72..652aba0e9477f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheReadFromBackupTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheReadFromBackupTest.java @@ -50,6 +50,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -87,7 +88,23 @@ public class IgniteCacheReadFromBackupTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testGetFromBackupStoreReadThroughEnabled() throws Exception { - for (CacheConfiguration ccfg : cacheConfigurations()) { + checkGetFromBackupStoreReadThroughEnabled(cacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + public void testMvccGetFromBackupStoreReadThroughEnabled() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10274"); + + checkGetFromBackupStoreReadThroughEnabled(mvccCacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + private void checkGetFromBackupStoreReadThroughEnabled(List> cacheCfgs) throws Exception { + for (CacheConfiguration ccfg : cacheCfgs) { ccfg.setCacheStoreFactory(new TestStoreFactory()); ccfg.setReadThrough(true); @@ -131,7 +148,23 @@ public void testGetFromBackupStoreReadThroughEnabled() throws Exception { * @throws Exception If failed. */ public void testGetFromBackupStoreReadThroughDisabled() throws Exception { - for (CacheConfiguration ccfg : cacheConfigurations()) { + checkGetFromBackupStoreReadThroughDisabled(cacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + public void testMvccGetFromBackupStoreReadThroughDisabled() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10274"); + + checkGetFromBackupStoreReadThroughDisabled(mvccCacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + private void checkGetFromBackupStoreReadThroughDisabled(List> cacheCfgs) throws Exception { + for (CacheConfiguration ccfg : cacheCfgs) { ccfg.setCacheStoreFactory(new TestStoreFactory()); ccfg.setReadThrough(false); @@ -159,7 +192,23 @@ public void testGetFromBackupStoreReadThroughDisabled() throws Exception { * @throws Exception If failed. */ public void testGetFromPrimaryPreloadInProgress() throws Exception { - for (final CacheConfiguration ccfg : cacheConfigurations()) { + checkGetFromPrimaryPreloadInProgress(cacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + public void testMvccGetFromPrimaryPreloadInProgress() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10274"); + + checkGetFromPrimaryPreloadInProgress(mvccCacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + private void checkGetFromPrimaryPreloadInProgress(List> cacheCfgs) throws Exception { + for (final CacheConfiguration ccfg : cacheCfgs) { boolean near = (ccfg.getNearConfiguration() != null); log.info("Test cache [mode=" + ccfg.getCacheMode() + @@ -245,7 +294,24 @@ public void testGetFromPrimaryPreloadInProgress() throws Exception { * @throws Exception If failed. */ public void testNoPrimaryReadPreloadFinished() throws Exception { - for (CacheConfiguration ccfg : cacheConfigurations()) { + checkNoPrimaryReadPreloadFinished(cacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + public void testMvccNoPrimaryReadPreloadFinished() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10274"); + + checkNoPrimaryReadPreloadFinished(mvccCacheConfigurations()); + + } + + /** + * @throws Exception If failed. + */ + private void checkNoPrimaryReadPreloadFinished(List> cacheCfgs) throws Exception { + for (CacheConfiguration ccfg : cacheCfgs) { boolean near = (ccfg.getNearConfiguration() != null); log.info("Test cache [mode=" + ccfg.getCacheMode() + @@ -369,6 +435,21 @@ private List> cacheConfigurations() { return ccfgs; } + /** + * @return Cache configurations to test. + */ + private List> mvccCacheConfigurations() { + List> ccfgs = new ArrayList<>(); + + ccfgs.add(cacheConfiguration(REPLICATED, TRANSACTIONAL_SNAPSHOT, 0, false)); + + ccfgs.add(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, false)); + ccfgs.add(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 1, true)); + ccfgs.add(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, 2, false)); + + return ccfgs; + } + /** * @param cacheMode Cache mode. * @param atomicityMode Cache atomicity mode. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSingleGetMessageTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSingleGetMessageTest.java index 974bcf21dc292..df6100bb52530 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSingleGetMessageTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheSingleGetMessageTest.java @@ -37,6 +37,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -88,11 +89,25 @@ public class IgniteCacheSingleGetMessageTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testSingleGetMessage() throws Exception { + checkSingleGetMessage(cacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + public void testMvccSingleGetMessage() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7371"); + + checkSingleGetMessage(mvccCacheConfigurations()); + } + + /** + * @throws Exception If failed. + */ + public void checkSingleGetMessage(List> ccfgs) throws Exception { assertFalse(ignite(0).configuration().isClientMode()); assertTrue(ignite(SRVS).configuration().isClientMode()); - List> ccfgs = cacheConfigurations(); - for (int i = 0; i < ccfgs.size(); i++) { CacheConfiguration ccfg = ccfgs.get(i); @@ -281,6 +296,19 @@ private List> cacheConfigurations() { return ccfgs; } + /** + * @return Mvcc cache configurations to test. + */ + private List> mvccCacheConfigurations() { + List> ccfgs = new ArrayList<>(); + + ccfgs.add(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, FULL_SYNC, 0)); + ccfgs.add(cacheConfiguration(PARTITIONED, TRANSACTIONAL_SNAPSHOT, FULL_SYNC, 1)); + ccfgs.add(cacheConfiguration(REPLICATED, TRANSACTIONAL_SNAPSHOT, FULL_SYNC, 0)); + + return ccfgs; + } + /** * @param cacheMode Cache mode. * @param atomicityMode Cache atomicity mode. @@ -297,7 +325,6 @@ private CacheConfiguration cacheConfiguration( ccfg.setCacheMode(cacheMode); ccfg.setAtomicityMode(atomicityMode); - ccfg.setAtomicityMode(TRANSACTIONAL); ccfg.setWriteSynchronizationMode(syncMode); if (cacheMode == PARTITIONED) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java index b4af2a8ed1d14..65e9f37174b92 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPreloadAbstractTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -78,6 +79,9 @@ public abstract class IgniteTxPreloadAbstractTest extends GridCacheAbstractSelfT * @throws Exception If failed. */ public void testRemoteTxPreloading() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + IgniteCache cache = jcache(0); for (int i = 0; i < 10_000; i++) @@ -147,13 +151,17 @@ public void testRemoteTxPreloading() throws Exception { * @throws Exception If failed. */ public void testLocalTxPreloadingOptimistic() throws Exception { - testLocalTxPreloading(OPTIMISTIC); + if (!MvccFeatureChecker.forcedMvcc()) // Do not check optimistic tx for mvcc. + testLocalTxPreloading(OPTIMISTIC); } /** * @throws Exception If failed. */ public void testLocalTxPreloadingPessimistic() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + testLocalTxPreloading(PESSIMISTIC); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheLockFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheLockFailoverSelfTest.java index f813ef89000fe..ac6ba5ef4726c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheLockFailoverSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheLockFailoverSelfTest.java @@ -31,12 +31,20 @@ import org.apache.ignite.lang.IgniteFutureTimeoutException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * */ @SuppressWarnings("unchecked") public class IgniteCacheLockFailoverSelfTest extends GridCacheAbstractSelfTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 2; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java index b5771054a56cf..65ab0c6d19dbd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java @@ -22,6 +22,7 @@ import java.util.TreeMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -34,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -56,6 +58,16 @@ public class IgniteCacheMultiTxLockSelfTest extends GridCommonAbstractTest { /** */ private boolean client; + /** Unexpected lock error. */ + private volatile Throwable err; + + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); @@ -121,6 +133,8 @@ public void testExplicitLockManyKeysWithClient() throws Exception { public void checkExplicitLock(int keys, boolean testClient) throws Exception { Collection threads = new ArrayList<>(); + err = null; + try { // Start grid 1. IgniteEx grid1 = startGrid(1); @@ -172,6 +186,8 @@ public void checkExplicitLock(int keys, boolean testClient) throws Exception { assertEquals("txMap is not empty:" + i, 0, tm.idMapSize()); } + + assertNull(err); } finally { stopAllGrids(); @@ -216,7 +232,7 @@ private Thread runCacheOperations(final IgniteInternalCache cache else cache.removeAll(vals.keySet()); } - catch (Exception e) { + catch (IgniteCheckedException e) { U.error(log(), "Failed cache operation.", e); } finally { @@ -225,8 +241,12 @@ private Thread runCacheOperations(final IgniteInternalCache cache U.sleep(100); } - catch (Exception e){ + catch (Throwable e){ U.error(log(), "Failed unlock.", e); + + err = e; + + return; } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheMvccTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheMvccTxSelfTest.java new file mode 100644 index 0000000000000..63a255e922d5d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheMvccTxSelfTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import org.apache.ignite.cache.CacheAtomicityMode; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; + +/** + * + */ +public class IgniteCrossCacheMvccTxSelfTest extends IgniteCrossCacheTxAbstractSelfTest { + /** {@inheritDoc} */ + @Override public CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + } + + /** + * @throws Exception If failed. + */ + public void testPessimisticRepeatableRead() throws Exception { + checkTxsSingleOp(PESSIMISTIC, REPEATABLE_READ); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxAbstractSelfTest.java new file mode 100644 index 0000000000000..5224daf532bda --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxAbstractSelfTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; + +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * Tests specific combinations of cross-cache transactions. + */ +public abstract class IgniteCrossCacheTxAbstractSelfTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static final String FIRST_CACHE = "FirstCache"; + + /** */ + private static final String SECOND_CACHE = "SecondCache"; + + /** */ + private static final int TX_CNT = 500; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + return cfg; + } + + /** + * @return Node count for this test. + */ + private int nodeCount() { + return 4; + } + + /** + * @return {@code True} if near cache should be enabled. + */ + protected boolean nearEnabled() { + return false; + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override protected void beforeTestsStarted() throws Exception { + startGridsMultiThreaded(nodeCount()); + + CacheConfiguration firstCfg = new CacheConfiguration(FIRST_CACHE); + firstCfg.setBackups(1); + firstCfg.setAtomicityMode(atomicityMode()); + firstCfg.setWriteSynchronizationMode(FULL_SYNC); + + grid(0).createCache(firstCfg); + + CacheConfiguration secondCfg = new CacheConfiguration(SECOND_CACHE); + secondCfg.setBackups(1); + secondCfg.setAtomicityMode(atomicityMode()); + secondCfg.setWriteSynchronizationMode(FULL_SYNC); + + if (nearEnabled()) + secondCfg.setNearConfiguration(new NearCacheConfiguration()); + + grid(0).createCache(secondCfg); + } + + /** + * @return Atomicity mode. + */ + public abstract CacheAtomicityMode atomicityMode(); + + /** + * @param concurrency Concurrency. + * @param isolation Isolation. + * @throws Exception If failed. + */ + protected void checkTxsSingleOp(TransactionConcurrency concurrency, TransactionIsolation isolation) throws Exception { + Map firstCheck = new HashMap<>(); + Map secondCheck = new HashMap<>(); + + for (int i = 0; i < TX_CNT; i++) { + int grid = ThreadLocalRandom.current().nextInt(nodeCount()); + + IgniteCache first = grid(grid).cache(FIRST_CACHE); + IgniteCache second = grid(grid).cache(SECOND_CACHE); + + try (Transaction tx = grid(grid).transactions().txStart(concurrency, isolation)) { + try { + int size = ThreadLocalRandom.current().nextInt(24) + 1; + + for (int k = 0; k < size; k++) { + boolean rnd = ThreadLocalRandom.current().nextBoolean(); + + IgniteCache cache = rnd ? first : second; + Map check = rnd ? firstCheck : secondCheck; + + String val = rnd ? "first" + i : "second" + i; + + cache.put(k, val); + check.put(k, val); + } + + tx.commit(); + } + catch (Throwable e) { + e.printStackTrace(); + + throw e; + } + } + + if (i > 0 && i % 100 == 0) + info("Finished iteration: " + i); + } + + for (int g = 0; g < nodeCount(); g++) { + IgniteEx grid = grid(g); + + assertEquals(0, grid.context().cache().context().tm().idMapSize()); + + ClusterNode locNode = grid.localNode(); + + IgniteCache firstCache = grid.cache(FIRST_CACHE); + + for (Map.Entry entry : firstCheck.entrySet()) { + boolean primary = grid.affinity(FIRST_CACHE).isPrimary(locNode, entry.getKey()); + + boolean backup = grid.affinity(FIRST_CACHE).isBackup(locNode, entry.getKey()); + + assertEquals("Invalid value found first cache [primary=" + primary + ", backup=" + backup + + ", node=" + locNode.id() + ", key=" + entry.getKey() + ']', + entry.getValue(), firstCache.get(entry.getKey())); + } + + for (Map.Entry entry : secondCheck.entrySet()) { + boolean primary = grid.affinity(SECOND_CACHE).isPrimary(locNode, entry.getKey()); + + boolean backup = grid.affinity(SECOND_CACHE).isBackup(locNode, entry.getKey()); + + assertEquals("Invalid value found second cache [primary=" + primary + ", backup=" + backup + + ", node=" + locNode.id() + ", key=" + entry.getKey() + ']', + entry.getValue(), grid.cache(SECOND_CACHE).get(entry.getKey())); + } + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxSelfTest.java index 91c66850a6f23..3b0106c4bc393 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCrossCacheTxSelfTest.java @@ -14,28 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.ignite.internal.processors.cache.distributed.dht; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.NearCacheConfiguration; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.apache.ignite.transactions.Transaction; -import org.apache.ignite.transactions.TransactionConcurrency; -import org.apache.ignite.transactions.TransactionIsolation; +import org.apache.ignite.cache.CacheAtomicityMode; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; @@ -43,65 +25,12 @@ import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; /** - * Tests specific combinations of cross-cache transactions. + * */ -public class IgniteCrossCacheTxSelfTest extends GridCommonAbstractTest { - /** */ - private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** */ - private static final String FIRST_CACHE = "FirstCache"; - - /** */ - private static final String SECOND_CACHE = "SecondCache"; - - /** */ - private static final int TX_CNT = 500; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); - - return cfg; - } - - /** - * @return Node count for this test. - */ - private int nodeCount() { - return 4; - } - - /** - * @return {@code True} if near cache should be enabled. - */ - protected boolean nearEnabled() { - return false; - } - +public class IgniteCrossCacheTxSelfTest extends IgniteCrossCacheTxAbstractSelfTest { /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override protected void beforeTestsStarted() throws Exception { - startGridsMultiThreaded(nodeCount()); - - CacheConfiguration firstCfg = new CacheConfiguration(FIRST_CACHE); - firstCfg.setBackups(1); - firstCfg.setAtomicityMode(TRANSACTIONAL); - firstCfg.setWriteSynchronizationMode(FULL_SYNC); - - grid(0).createCache(firstCfg); - - CacheConfiguration secondCfg = new CacheConfiguration(SECOND_CACHE); - secondCfg.setBackups(1); - secondCfg.setAtomicityMode(TRANSACTIONAL); - secondCfg.setWriteSynchronizationMode(FULL_SYNC); - - if (nearEnabled()) - secondCfg.setNearConfiguration(new NearCacheConfiguration()); - - grid(0).createCache(secondCfg); + @Override public CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.TRANSACTIONAL; } /** @@ -139,78 +68,4 @@ public void testOptimisticSerializable() throws Exception { checkTxsSingleOp(OPTIMISTIC, SERIALIZABLE); } - /** - * @param concurrency Concurrency. - * @param isolation Isolation. - * @throws Exception If failed. - */ - private void checkTxsSingleOp(TransactionConcurrency concurrency, TransactionIsolation isolation) throws Exception { - Map firstCheck = new HashMap<>(); - Map secondCheck = new HashMap<>(); - - for (int i = 0; i < TX_CNT; i++) { - int grid = ThreadLocalRandom.current().nextInt(nodeCount()); - - IgniteCache first = grid(grid).cache(FIRST_CACHE); - IgniteCache second = grid(grid).cache(SECOND_CACHE); - - try (Transaction tx = grid(grid).transactions().txStart(concurrency, isolation)) { - try { - int size = ThreadLocalRandom.current().nextInt(24) + 1; - - for (int k = 0; k < size; k++) { - boolean rnd = ThreadLocalRandom.current().nextBoolean(); - - IgniteCache cache = rnd ? first : second; - Map check = rnd ? firstCheck : secondCheck; - - String val = rnd ? "first" + i : "second" + i; - - cache.put(k, val); - check.put(k, val); - } - - tx.commit(); - } - catch (Throwable e) { - e.printStackTrace(); - - throw e; - } - } - - if (i > 0 && i % 100 == 0) - info("Finished iteration: " + i); - } - - for (int g = 0; g < nodeCount(); g++) { - IgniteEx grid = grid(g); - - assertEquals(0, grid.context().cache().context().tm().idMapSize()); - - ClusterNode locNode = grid.localNode(); - - IgniteCache firstCache = grid.cache(FIRST_CACHE); - - for (Map.Entry entry : firstCheck.entrySet()) { - boolean primary = grid.affinity(FIRST_CACHE).isPrimary(locNode, entry.getKey()); - - boolean backup = grid.affinity(FIRST_CACHE).isBackup(locNode, entry.getKey()); - - assertEquals("Invalid value found first cache [primary=" + primary + ", backup=" + backup + - ", node=" + locNode.id() + ", key=" + entry.getKey() + ']', - entry.getValue(), firstCache.get(entry.getKey())); - } - - for (Map.Entry entry : secondCheck.entrySet()) { - boolean primary = grid.affinity(SECOND_CACHE).isPrimary(locNode, entry.getKey()); - - boolean backup = grid.affinity(SECOND_CACHE).isBackup(locNode, entry.getKey()); - - assertEquals("Invalid value found second cache [primary=" + primary + ", backup=" + backup + - ", node=" + locNode.id() + ", key=" + entry.getKey() + ']', - entry.getValue(), grid.cache(SECOND_CACHE).get(entry.getKey())); - } - } - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxPreloadSelfTest.java index caf6cc7c7e74e..96b932a4a73d0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxPreloadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxPreloadSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxPreloadAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -28,6 +29,13 @@ * Tests cache transaction during preloading. */ public class GridCacheNearTxPreloadSelfTest extends IgniteTxPreloadAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheMode cacheMode() { return PARTITIONED; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java index 1c79db7e7a111..54eb32d3f1ce6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedEventSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.distributed.GridCacheEventAbstractTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -30,6 +31,13 @@ * Tests events. */ public class GridCachePartitionedEventSelfTest extends GridCacheEventAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_EVENTS); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { CacheConfiguration cfg = defaultCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearOnlyTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearOnlyTxTest.java index ca12a9925ca99..d86fa91a376a1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearOnlyTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearOnlyTxTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -40,6 +41,13 @@ * */ public class IgniteCacheNearOnlyTxTest extends IgniteCacheAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 2; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearReadCommittedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearReadCommittedTest.java index ad9bce047b2b2..76f0d178323c4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearReadCommittedTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheNearReadCommittedTest.java @@ -22,6 +22,7 @@ import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -33,6 +34,13 @@ */ @SuppressWarnings("RedundantMethodOverride") public class IgniteCacheNearReadCommittedTest extends GridCacheAbstractSelfTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 2; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheLoadAllAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheLoadAllAbstractTest.java index 6b4f9761311ca..3d9b7bd5c81dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheLoadAllAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheLoadAllAbstractTest.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import javax.cache.Cache; import javax.cache.configuration.Factory; import javax.cache.integration.CacheLoader; @@ -34,7 +35,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest; -import java.util.concurrent.ConcurrentHashMap; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Test for {@link Cache#loadAll(Set, boolean, CompletionListener)}. @@ -46,6 +47,13 @@ public abstract class IgniteCacheLoadAllAbstractTest extends IgniteCacheAbstract /** */ private static ConcurrentHashMap storeMap; + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLoaderWriterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLoaderWriterTest.java index fc4ec781a80ba..75a4cd93fb478 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLoaderWriterTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLoaderWriterTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxLoaderWriterTest extends IgniteCacheLoaderWriterAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 3; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalLoadAllTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalLoadAllTest.java index 97374bf3304e0..645a9ff3ad2c5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalLoadAllTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalLoadAllTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,12 @@ * */ public class IgniteCacheTxLocalLoadAllTest extends IgniteCacheLoadAllAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoLoadPreviousValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoLoadPreviousValueTest.java index 686b4ca310fb0..2e52a63a3aac3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoLoadPreviousValueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoLoadPreviousValueTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxLocalNoLoadPreviousValueTest extends IgniteCacheNoLoadPreviousValueAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoReadThroughTest.java index 5235c9d454b1b..e25a96516c9e2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoReadThroughTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxLocalNoReadThroughTest extends IgniteCacheNoReadThroughAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoWriteThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoWriteThroughTest.java index 7985c57207122..ddc9352655758 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoWriteThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxLocalNoWriteThroughTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxLocalNoWriteThroughTest extends IgniteCacheNoWriteThroughAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoLoadPreviousValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoLoadPreviousValueTest.java index ce42e393647b7..c343402c5769d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoLoadPreviousValueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoLoadPreviousValueTest.java @@ -18,11 +18,19 @@ package org.apache.ignite.internal.processors.cache.integration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * */ public class IgniteCacheTxNearEnabledNoLoadPreviousValueTest extends IgniteCacheTxNoLoadPreviousValueTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoWriteThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoWriteThroughTest.java index ea12cce995bea..be9bc343f09c5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoWriteThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNearEnabledNoWriteThroughTest.java @@ -18,11 +18,19 @@ package org.apache.ignite.internal.processors.cache.integration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * */ public class IgniteCacheTxNearEnabledNoWriteThroughTest extends IgniteCacheTxNoWriteThroughTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoLoadPreviousValueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoLoadPreviousValueTest.java index ce9ef9b467ab2..094aa1a2c857f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoLoadPreviousValueTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoLoadPreviousValueTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxNoLoadPreviousValueTest extends IgniteCacheNoLoadPreviousValueAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 3; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoReadThroughTest.java index 90bc768ed60bf..a94caff0935a7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoReadThroughTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxNoReadThroughTest extends IgniteCacheNoReadThroughAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 3; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoWriteThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoWriteThroughTest.java index 325d96994d51f..e984dfb4ba11c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoWriteThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxNoWriteThroughTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -28,6 +29,13 @@ * */ public class IgniteCacheTxNoWriteThroughTest extends IgniteCacheNoWriteThroughAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 3; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionTest.java index b9e884b9b4d84..77f54af47f5a6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; @@ -38,6 +39,13 @@ * */ public class IgniteCacheTxStoreSessionTest extends IgniteCacheStoreSessionAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected int gridCount() { return 3; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java index a90b4f18cf04d..ea2b8a3cc42de 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java @@ -22,6 +22,7 @@ import javax.cache.integration.CacheWriterException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -30,6 +31,13 @@ * parameter. */ public class IgniteCacheTxStoreSessionWriteBehindCoalescingTest extends IgniteCacheStoreSessionWriteBehindAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindTest.java index b72aba39b3492..ae6bcae84d9b2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.integration; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -25,6 +26,13 @@ * */ public class IgniteCacheTxStoreSessionWriteBehindTest extends IgniteCacheStoreSessionWriteBehindAbstractTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.setUp(); + } + /** {@inheritDoc} */ @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryLocalTransactionalSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryLocalTransactionalSelfTest.java index d7fc93809848d..b217703f28a6a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryLocalTransactionalSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryLocalTransactionalSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * @@ -38,4 +39,12 @@ public class CacheVersionedEntryLocalTransactionalSelfTest extends CacheVersione @Override protected CacheAtomicityMode atomicityMode() { return CacheAtomicityMode.TRANSACTIONAL; } + + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.setUp(); + } + } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index 183c2eb6e82ff..c70d447c49254 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -19,10 +19,15 @@ import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; import static org.apache.ignite.IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS; +import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.TRANSACTION_SERIALIZATION_ERROR; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; /** @@ -106,12 +111,27 @@ public static boolean isSupported(CacheMode mode) { return mode != CacheMode.LOCAL || isSupported(Feature.LOCAL_CACHE); } + /** + * TODO proper exception handling after https://issues.apache.org/jira/browse/IGNITE-9470 + * Checks if given exception was caused by MVCC write conflict. + * + * @param e Exception. + */ + public static void assertMvccWriteConflict(Exception e) { + IgniteSQLException sqlEx = X.cause(e, IgniteSQLException.class); + + assertNotNull(sqlEx); + + assertEquals(TRANSACTION_SERIALIZATION_ERROR, sqlEx.statusCode()); + } + /** * Fails if feature is not supported in Mvcc mode. * * @param feature Mvcc feature. * @throws AssertionError If failed. */ + @SuppressWarnings("fallthrough") private static void validateFeature(Feature feature) { switch (feature) { case NEAR_CACHE: diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java index 581865309fc69..3c070bc033e10 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite2.java @@ -115,6 +115,7 @@ public static TestSuite suite() { ignoredTests.add(CacheOptimisticTransactionsWithFilterSingleServerTest.class); ignoredTests.add(CacheOptimisticTransactionsWithFilterTest.class); + // Irrelevant Tx tests. ignoredTests.add(IgniteOnePhaseCommitInvokeTest.class); ignoredTests.add(IgniteOnePhaseCommitNearReadersTest.class); ignoredTests.add(GridCacheDhtPreloadOnheapSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite4.java new file mode 100644 index 0000000000000..99336337e71bd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite4.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.store.CacheStoreListenerRWThroughDisabledAtomicCacheTest; +import org.apache.ignite.internal.processors.cache.CacheConnectionLeakStoreTxTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticReadCommittedSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticRepeatableReadSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticSerializableSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticReadCommittedSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticRepeatableReadSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticSerializableSelfTest; +import org.apache.ignite.internal.processors.cache.CacheOffheapMapEntrySelfTest; +import org.apache.ignite.internal.processors.cache.CacheReadThroughAtomicRestartSelfTest; +import org.apache.ignite.internal.processors.cache.CacheReadThroughLocalAtomicRestartSelfTest; +import org.apache.ignite.internal.processors.cache.CacheReadThroughReplicatedAtomicRestartSelfTest; +import org.apache.ignite.internal.processors.cache.CacheStoreUsageMultinodeDynamicStartAtomicTest; +import org.apache.ignite.internal.processors.cache.CacheStoreUsageMultinodeStaticStartAtomicTest; +import org.apache.ignite.internal.processors.cache.CacheTxNotAllowReadFromBackupTest; +import org.apache.ignite.internal.processors.cache.GridCacheMultinodeUpdateAtomicNearEnabledSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheMultinodeUpdateAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheVersionMultinodeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicCopyOnReadDisabledTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalPeekModesTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalStoreValueTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicNearEnabledStoreValueTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicNearPeekModesTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicPeekModesTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicReplicatedPeekModesTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicStoreValueTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheConfigurationDefaultTemplateTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheContainsKeyAtomicTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheInvokeReadThroughSingleNodeTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheInvokeReadThroughTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheStartTest; +import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest; +import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest; +import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartStopConcurrentTest; +import org.apache.ignite.internal.processors.cache.IgniteDynamicClientCacheStartSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteExchangeFutureHistoryTest; +import org.apache.ignite.internal.processors.cache.IgniteInternalCacheTypesTest; +import org.apache.ignite.internal.processors.cache.IgniteStartCacheInTransactionAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest; +import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheDiscoveryDataConcurrentJoinTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheGroupsPreloadTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheResultIsNotNullOnPartitionLossTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheFailedUpdateResponseTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheReadFromBackupTest; +import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheSingleGetMessageTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCrossCacheMvccTxSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCrossCacheTxSelfTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLoadAllTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLoaderWriterTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLocalLoadAllTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLocalNoLoadPreviousValueTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLocalNoReadThroughTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicLocalNoWriteThroughTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicNearEnabledNoLoadPreviousValueTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicNearEnabledNoReadThroughTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicNearEnabledNoWriteThroughTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicNoLoadPreviousValueTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicNoReadThroughTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicNoWriteThroughTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicStoreSessionTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheAtomicStoreSessionWriteBehindTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheJdbcBlobStoreNodeRestartTest; +import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryLocalAtomicSwapDisabledSelfTest; +import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryPartitionedAtomicSelfTest; +import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedAtomicSelfTest; + +/** + * + */ +public class IgniteCacheMvccTestSuite4 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + HashSet ignoredTests = new HashSet<>(128); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(GridCacheVersionMultinodeTest.class); + ignoredTests.add(IgniteCacheCreatePutTest.class); + ignoredTests.add(IgniteClientCacheInitializationFailTest.class); + ignoredTests.add(IgniteCacheFailedUpdateResponseTest.class); + ignoredTests.add(CacheGetEntryPessimisticRepeatableReadSelfTest.class); + ignoredTests.add(CacheTxNotAllowReadFromBackupTest.class); + ignoredTests.add(CacheOffheapMapEntrySelfTest.class); + ignoredTests.add(CacheGroupsPreloadTest.class); + ignoredTests.add(CacheConnectionLeakStoreTxTest.class); + ignoredTests.add(IgniteCacheInvokeReadThroughTest.class); + ignoredTests.add(IgniteCacheInvokeReadThroughSingleNodeTest.class); + ignoredTests.add(IgniteDynamicCacheStartSelfTest.class); + ignoredTests.add(IgniteDynamicClientCacheStartSelfTest.class); + ignoredTests.add(IgniteDynamicCacheStartNoExchangeTimeoutTest.class); + ignoredTests.add(IgniteCacheSingleGetMessageTest.class); + ignoredTests.add(IgniteCacheReadFromBackupTest.class); + + // Optimistic tx tests. + ignoredTests.add(CacheGetEntryOptimisticReadCommittedSelfTest.class); + ignoredTests.add(CacheGetEntryOptimisticRepeatableReadSelfTest.class); + ignoredTests.add(CacheGetEntryOptimisticSerializableSelfTest.class); + + // Irrelevant Tx tests. + ignoredTests.add(CacheGetEntryPessimisticReadCommittedSelfTest.class); + ignoredTests.add(CacheGetEntryPessimisticSerializableSelfTest.class); + + // Atomic cache tests. + ignoredTests.add(GridCacheMultinodeUpdateAtomicSelfTest.class); + ignoredTests.add(GridCacheMultinodeUpdateAtomicNearEnabledSelfTest.class); + ignoredTests.add(IgniteCacheAtomicLoadAllTest.class); + ignoredTests.add(IgniteCacheAtomicLocalLoadAllTest.class); + ignoredTests.add(IgniteCacheAtomicLoaderWriterTest.class); + ignoredTests.add(IgniteCacheAtomicStoreSessionTest.class); + ignoredTests.add(IgniteCacheAtomicStoreSessionWriteBehindTest.class); + ignoredTests.add(IgniteCacheAtomicNoReadThroughTest.class); + ignoredTests.add(IgniteCacheAtomicNearEnabledNoReadThroughTest.class); + ignoredTests.add(IgniteCacheAtomicLocalNoReadThroughTest.class); + ignoredTests.add(IgniteCacheAtomicNoLoadPreviousValueTest.class); + ignoredTests.add(IgniteCacheAtomicNearEnabledNoLoadPreviousValueTest.class); + ignoredTests.add(IgniteCacheAtomicLocalNoLoadPreviousValueTest.class); + ignoredTests.add(IgniteCacheAtomicNoWriteThroughTest.class); + ignoredTests.add(IgniteCacheAtomicNearEnabledNoWriteThroughTest.class); + ignoredTests.add(IgniteCacheAtomicLocalNoWriteThroughTest.class); + ignoredTests.add(IgniteCacheAtomicPeekModesTest.class); + ignoredTests.add(IgniteCacheAtomicNearPeekModesTest.class); + ignoredTests.add(IgniteCacheAtomicReplicatedPeekModesTest.class); + ignoredTests.add(IgniteCacheAtomicLocalPeekModesTest.class); + ignoredTests.add(IgniteCacheAtomicCopyOnReadDisabledTest.class); + ignoredTests.add(IgniteCacheAtomicLocalStoreValueTest.class); + ignoredTests.add(IgniteCacheAtomicStoreValueTest.class); + ignoredTests.add(IgniteCacheAtomicNearEnabledStoreValueTest.class); + ignoredTests.add(CacheStoreListenerRWThroughDisabledAtomicCacheTest.class); + ignoredTests.add(CacheStoreUsageMultinodeStaticStartAtomicTest.class); + ignoredTests.add(CacheStoreUsageMultinodeDynamicStartAtomicTest.class); + ignoredTests.add(IgniteStartCacheInTransactionAtomicSelfTest.class); + ignoredTests.add(CacheReadThroughReplicatedAtomicRestartSelfTest.class); + ignoredTests.add(CacheReadThroughLocalAtomicRestartSelfTest.class); + ignoredTests.add(CacheReadThroughAtomicRestartSelfTest.class); + ignoredTests.add(CacheVersionedEntryLocalAtomicSwapDisabledSelfTest.class); + ignoredTests.add(CacheVersionedEntryPartitionedAtomicSelfTest.class); + ignoredTests.add(CacheGetFutureHangsSelfTest.class); + ignoredTests.add(IgniteCacheContainsKeyAtomicTest.class); + ignoredTests.add(CacheVersionedEntryReplicatedAtomicSelfTest.class); + ignoredTests.add(CacheResultIsNotNullOnPartitionLossTest.class); + + // Other non-tx tests. + ignoredTests.add(IgniteDynamicCacheStartStopConcurrentTest.class); + ignoredTests.add(IgniteCacheConfigurationDefaultTemplateTest.class); + ignoredTests.add(IgniteCacheStartTest.class); + ignoredTests.add(CacheDiscoveryDataConcurrentJoinTest.class); + ignoredTests.add(IgniteCacheJdbcBlobStoreNodeRestartTest.class); + ignoredTests.add(IgniteInternalCacheTypesTest.class); + ignoredTests.add(IgniteExchangeFutureHistoryTest.class); + ignoredTests.add(CacheNoValueClassOnServerNodeTest.class); + ignoredTests.add(IgniteSystemCacheOnClientTest.class); + ignoredTests.add(MarshallerCacheJobRunNodeRestartTest.class); + + // Skip classes which Mvcc implementations are added in this method below. + // TODO IGNITE-10175: refactor these tests (use assume) to support both mvcc and non-mvcc modes after moving to JUnit4/5. + ignoredTests.add(IgniteCrossCacheTxSelfTest.class); + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 4"); + + suite.addTest(IgniteCacheTestSuite4.suite(ignoredTests)); + + // Add Mvcc clones. + suite.addTestSuite(IgniteCrossCacheMvccTxSelfTest.class); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java index 8ea329ee0c0ce..04620c8f7fed1 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http:www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.HashSet; import junit.framework.TestSuite; import org.apache.ignite.cache.store.CacheStoreListenerRWThroughDisabledAtomicCacheTest; import org.apache.ignite.cache.store.CacheStoreListenerRWThroughDisabledTransactionalCacheTest; @@ -26,12 +27,12 @@ import org.apache.ignite.internal.processors.GridCacheTxLoadFromStoreOnLockSelfTest; import org.apache.ignite.internal.processors.cache.CacheClientStoreSelfTest; import org.apache.ignite.internal.processors.cache.CacheConnectionLeakStoreTxTest; -import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticReadCommittedSeltTest; -import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticRepeatableReadSeltTest; -import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticSerializableSeltTest; -import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticReadCommittedSeltTest; -import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticRepeatableReadSeltTest; -import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticSerializableSeltTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticReadCommittedSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticRepeatableReadSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryOptimisticSerializableSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticReadCommittedSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticRepeatableReadSelfTest; +import org.apache.ignite.internal.processors.cache.CacheGetEntryPessimisticSerializableSelfTest; import org.apache.ignite.internal.processors.cache.CacheOffheapMapEntrySelfTest; import org.apache.ignite.internal.processors.cache.CacheReadThroughAtomicRestartSelfTest; import org.apache.ignite.internal.processors.cache.CacheReadThroughLocalAtomicRestartSelfTest; @@ -154,191 +155,200 @@ import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedAtomicSelfTest; import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedTransactionalSelfTest; +import static org.apache.ignite.testframework.GridTestUtils.addTestIfNeeded; + /** * Test suite. */ public class IgniteCacheTestSuite4 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @param ignoredTests Ignored tests. + * @return IgniteCache test suite. + */ + public static TestSuite suite(HashSet ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 4"); // Multi node update. - suite.addTestSuite(GridCacheMultinodeUpdateSelfTest.class); - suite.addTestSuite(GridCacheMultinodeUpdateNearEnabledSelfTest.class); - suite.addTestSuite(GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest.class); - suite.addTestSuite(GridCacheMultinodeUpdateAtomicSelfTest.class); - suite.addTestSuite(GridCacheMultinodeUpdateAtomicNearEnabledSelfTest.class); - - suite.addTestSuite(IgniteCacheAtomicLoadAllTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalLoadAllTest.class); - suite.addTestSuite(IgniteCacheTxLoadAllTest.class); - suite.addTestSuite(IgniteCacheTxLocalLoadAllTest.class); - - suite.addTestSuite(IgniteCacheAtomicLoaderWriterTest.class); - suite.addTestSuite(IgniteCacheTxLoaderWriterTest.class); - - suite.addTestSuite(IgniteCacheAtomicStoreSessionTest.class); - suite.addTestSuite(IgniteCacheTxStoreSessionTest.class); - suite.addTestSuite(IgniteCacheAtomicStoreSessionWriteBehindTest.class); - suite.addTestSuite(IgniteCacheTxStoreSessionWriteBehindTest.class); - suite.addTestSuite(IgniteCacheTxStoreSessionWriteBehindCoalescingTest.class); - - suite.addTestSuite(IgniteCacheAtomicNoReadThroughTest.class); - suite.addTestSuite(IgniteCacheAtomicNearEnabledNoReadThroughTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalNoReadThroughTest.class); - suite.addTestSuite(IgniteCacheTxNoReadThroughTest.class); - suite.addTestSuite(IgniteCacheTxNearEnabledNoReadThroughTest.class); - suite.addTestSuite(IgniteCacheTxLocalNoReadThroughTest.class); - - suite.addTestSuite(IgniteCacheAtomicNoLoadPreviousValueTest.class); - suite.addTestSuite(IgniteCacheAtomicNearEnabledNoLoadPreviousValueTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalNoLoadPreviousValueTest.class); - suite.addTestSuite(IgniteCacheTxNoLoadPreviousValueTest.class); - suite.addTestSuite(IgniteCacheTxNearEnabledNoLoadPreviousValueTest.class); - suite.addTestSuite(IgniteCacheTxLocalNoLoadPreviousValueTest.class); - - suite.addTestSuite(IgniteCacheAtomicNoWriteThroughTest.class); - suite.addTestSuite(IgniteCacheAtomicNearEnabledNoWriteThroughTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalNoWriteThroughTest.class); - suite.addTestSuite(IgniteCacheTxNoWriteThroughTest.class); - suite.addTestSuite(IgniteCacheTxNearEnabledNoWriteThroughTest.class); - suite.addTestSuite(IgniteCacheTxLocalNoWriteThroughTest.class); - - suite.addTestSuite(IgniteCacheAtomicPeekModesTest.class); - suite.addTestSuite(IgniteCacheAtomicNearPeekModesTest.class); - suite.addTestSuite(IgniteCacheAtomicReplicatedPeekModesTest.class); - suite.addTestSuite(IgniteCacheAtomicLocalPeekModesTest.class); - suite.addTestSuite(IgniteCacheTxPeekModesTest.class); - suite.addTestSuite(IgniteCacheTxNearPeekModesTest.class); - suite.addTestSuite(IgniteCacheTxLocalPeekModesTest.class); - suite.addTestSuite(IgniteCacheTxReplicatedPeekModesTest.class); - - suite.addTestSuite(IgniteCacheInvokeReadThroughSingleNodeTest.class); - suite.addTestSuite(IgniteCacheInvokeReadThroughTest.class); - suite.addTestSuite(IgniteCacheReadThroughStoreCallTest.class); - suite.addTestSuite(GridCacheVersionMultinodeTest.class); - - suite.addTestSuite(IgniteCacheNearReadCommittedTest.class); - suite.addTestSuite(IgniteCacheAtomicCopyOnReadDisabledTest.class); - suite.addTestSuite(IgniteCacheTxCopyOnReadDisabledTest.class); - - suite.addTestSuite(IgniteCacheTxPreloadNoWriteTest.class); - - suite.addTestSuite(IgniteDynamicCacheStartSelfTest.class); - suite.addTestSuite(IgniteDynamicCacheMultinodeTest.class); - suite.addTestSuite(IgniteDynamicCacheStartFailTest.class); - suite.addTestSuite(IgniteDynamicCacheStartCoordinatorFailoverTest.class); - suite.addTestSuite(IgniteDynamicCacheWithConfigStartSelfTest.class); - suite.addTestSuite(IgniteCacheDynamicStopSelfTest.class); - suite.addTestSuite(IgniteDynamicCacheStartStopConcurrentTest.class); - suite.addTestSuite(IgniteCacheConfigurationTemplateTest.class); - suite.addTestSuite(IgniteCacheConfigurationDefaultTemplateTest.class); - suite.addTestSuite(IgniteDynamicClientCacheStartSelfTest.class); - suite.addTestSuite(IgniteDynamicCacheStartNoExchangeTimeoutTest.class); - suite.addTestSuite(CacheAffinityEarlyTest.class); - suite.addTestSuite(IgniteCacheCreatePutMultiNodeSelfTest.class); - suite.addTestSuite(IgniteCacheCreatePutTest.class); - suite.addTestSuite(CacheStartOnJoinTest.class); - suite.addTestSuite(IgniteCacheStartTest.class); - suite.addTestSuite(CacheDiscoveryDataConcurrentJoinTest.class); - suite.addTestSuite(IgniteClientCacheInitializationFailTest.class); - suite.addTestSuite(IgniteCacheFailedUpdateResponseTest.class); - - suite.addTestSuite(GridCacheTxLoadFromStoreOnLockSelfTest.class); - - suite.addTestSuite(GridCacheMarshallingNodeJoinSelfTest.class); - - suite.addTestSuite(IgniteCacheJdbcBlobStoreNodeRestartTest.class); - - suite.addTestSuite(IgniteCacheAtomicLocalStoreValueTest.class); - suite.addTestSuite(IgniteCacheAtomicStoreValueTest.class); - suite.addTestSuite(IgniteCacheAtomicNearEnabledStoreValueTest.class); - suite.addTestSuite(IgniteCacheTxLocalStoreValueTest.class); - suite.addTestSuite(IgniteCacheTxStoreValueTest.class); - suite.addTestSuite(IgniteCacheTxNearEnabledStoreValueTest.class); - - suite.addTestSuite(IgniteCacheLockFailoverSelfTest.class); - suite.addTestSuite(IgniteCacheMultiTxLockSelfTest.class); - - suite.addTestSuite(IgniteInternalCacheTypesTest.class); - - suite.addTestSuite(IgniteExchangeFutureHistoryTest.class); - - suite.addTestSuite(CacheNoValueClassOnServerNodeTest.class); - suite.addTestSuite(IgniteSystemCacheOnClientTest.class); - - suite.addTestSuite(CacheRemoveAllSelfTest.class); - suite.addTestSuite(CacheGetEntryOptimisticReadCommittedSeltTest.class); - suite.addTestSuite(CacheGetEntryOptimisticRepeatableReadSeltTest.class); - suite.addTestSuite(CacheGetEntryOptimisticSerializableSeltTest.class); - suite.addTestSuite(CacheGetEntryPessimisticReadCommittedSeltTest.class); - suite.addTestSuite(CacheGetEntryPessimisticRepeatableReadSeltTest.class); - suite.addTestSuite(CacheGetEntryPessimisticSerializableSeltTest.class); - suite.addTestSuite(CacheTxNotAllowReadFromBackupTest.class); - - suite.addTestSuite(CacheStopAndDestroySelfTest.class); - - suite.addTestSuite(CacheOffheapMapEntrySelfTest.class); - - suite.addTestSuite(CacheJdbcStoreSessionListenerSelfTest.class); - suite.addTestSuite(CacheStoreSessionListenerLifecycleSelfTest.class); - suite.addTestSuite(CacheStoreListenerRWThroughDisabledAtomicCacheTest.class); - suite.addTestSuite(CacheStoreListenerRWThroughDisabledTransactionalCacheTest.class); - suite.addTestSuite(CacheStoreSessionListenerWriteBehindEnabledTest.class); - - suite.addTestSuite(CacheClientStoreSelfTest.class); - suite.addTestSuite(CacheStoreUsageMultinodeStaticStartAtomicTest.class); - suite.addTestSuite(CacheStoreUsageMultinodeStaticStartTxTest.class); - suite.addTestSuite(CacheStoreUsageMultinodeDynamicStartAtomicTest.class); - suite.addTestSuite(CacheStoreUsageMultinodeDynamicStartTxTest.class); - suite.addTestSuite(CacheConnectionLeakStoreTxTest.class); - - suite.addTestSuite(GridCacheStoreManagerDeserializationTest.class); - suite.addTestSuite(GridLocalCacheStoreManagerDeserializationTest.class); - - suite.addTestSuite(IgniteStartCacheInTransactionSelfTest.class); - suite.addTestSuite(IgniteStartCacheInTransactionAtomicSelfTest.class); - - suite.addTestSuite(CacheReadThroughRestartSelfTest.class); - suite.addTestSuite(CacheReadThroughReplicatedRestartSelfTest.class); - suite.addTestSuite(CacheReadThroughReplicatedAtomicRestartSelfTest.class); - suite.addTestSuite(CacheReadThroughLocalRestartSelfTest.class); - suite.addTestSuite(CacheReadThroughLocalAtomicRestartSelfTest.class); - suite.addTestSuite(CacheReadThroughAtomicRestartSelfTest.class); + addTestIfNeeded(suite, GridCacheMultinodeUpdateSelfTest.class, ignoredTests); + addTestIfNeeded(suite, GridCacheMultinodeUpdateNearEnabledSelfTest.class, ignoredTests); + addTestIfNeeded(suite, GridCacheMultinodeUpdateNearEnabledNoBackupsSelfTest.class, ignoredTests); + addTestIfNeeded(suite, GridCacheMultinodeUpdateAtomicSelfTest.class, ignoredTests); + addTestIfNeeded(suite, GridCacheMultinodeUpdateAtomicNearEnabledSelfTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicLoadAllTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicLocalLoadAllTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLoadAllTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLocalLoadAllTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicLoaderWriterTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLoaderWriterTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicStoreSessionTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxStoreSessionTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicStoreSessionWriteBehindTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxStoreSessionWriteBehindTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxStoreSessionWriteBehindCoalescingTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicNoReadThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicNearEnabledNoReadThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicLocalNoReadThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNoReadThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNearEnabledNoReadThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLocalNoReadThroughTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicNoLoadPreviousValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicNearEnabledNoLoadPreviousValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicLocalNoLoadPreviousValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNoLoadPreviousValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNearEnabledNoLoadPreviousValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLocalNoLoadPreviousValueTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicNoWriteThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicNearEnabledNoWriteThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicLocalNoWriteThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNoWriteThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNearEnabledNoWriteThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLocalNoWriteThroughTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicNearPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicReplicatedPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicLocalPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNearPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLocalPeekModesTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxReplicatedPeekModesTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheInvokeReadThroughSingleNodeTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheInvokeReadThroughTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheReadThroughStoreCallTest.class, ignoredTests); + addTestIfNeeded(suite, GridCacheVersionMultinodeTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheNearReadCommittedTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicCopyOnReadDisabledTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxCopyOnReadDisabledTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheTxPreloadNoWriteTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteDynamicCacheStartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicCacheMultinodeTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicCacheStartFailTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicCacheStartCoordinatorFailoverTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicCacheWithConfigStartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheDynamicStopSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicCacheStartStopConcurrentTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheConfigurationTemplateTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheConfigurationDefaultTemplateTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicClientCacheStartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteDynamicCacheStartNoExchangeTimeoutTest.class, ignoredTests); + addTestIfNeeded(suite, CacheAffinityEarlyTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheCreatePutMultiNodeSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheCreatePutTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStartOnJoinTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheStartTest.class, ignoredTests); + addTestIfNeeded(suite, CacheDiscoveryDataConcurrentJoinTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteClientCacheInitializationFailTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheFailedUpdateResponseTest.class, ignoredTests); + + addTestIfNeeded(suite, GridCacheTxLoadFromStoreOnLockSelfTest.class, ignoredTests); + + addTestIfNeeded(suite, GridCacheMarshallingNodeJoinSelfTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheJdbcBlobStoreNodeRestartTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheAtomicLocalStoreValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicStoreValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheAtomicNearEnabledStoreValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxLocalStoreValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxStoreValueTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheTxNearEnabledStoreValueTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteCacheLockFailoverSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheMultiTxLockSelfTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteInternalCacheTypesTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteExchangeFutureHistoryTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheNoValueClassOnServerNodeTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteSystemCacheOnClientTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheRemoveAllSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGetEntryOptimisticReadCommittedSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGetEntryOptimisticRepeatableReadSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGetEntryOptimisticSerializableSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGetEntryPessimisticReadCommittedSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGetEntryPessimisticRepeatableReadSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGetEntryPessimisticSerializableSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheTxNotAllowReadFromBackupTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheStopAndDestroySelfTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheOffheapMapEntrySelfTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheJdbcStoreSessionListenerSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreSessionListenerLifecycleSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreListenerRWThroughDisabledAtomicCacheTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreListenerRWThroughDisabledTransactionalCacheTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreSessionListenerWriteBehindEnabledTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheClientStoreSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreUsageMultinodeStaticStartAtomicTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreUsageMultinodeStaticStartTxTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreUsageMultinodeDynamicStartAtomicTest.class, ignoredTests); + addTestIfNeeded(suite, CacheStoreUsageMultinodeDynamicStartTxTest.class, ignoredTests); + addTestIfNeeded(suite, CacheConnectionLeakStoreTxTest.class, ignoredTests); + + addTestIfNeeded(suite, GridCacheStoreManagerDeserializationTest.class, ignoredTests); + addTestIfNeeded(suite, GridLocalCacheStoreManagerDeserializationTest.class, ignoredTests); + + addTestIfNeeded(suite, IgniteStartCacheInTransactionSelfTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteStartCacheInTransactionAtomicSelfTest.class, ignoredTests); + + addTestIfNeeded(suite, CacheReadThroughRestartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheReadThroughReplicatedRestartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheReadThroughReplicatedAtomicRestartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheReadThroughLocalRestartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheReadThroughLocalAtomicRestartSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheReadThroughAtomicRestartSelfTest.class, ignoredTests); // Versioned entry tests - suite.addTestSuite(CacheVersionedEntryLocalAtomicSwapDisabledSelfTest.class); - suite.addTestSuite(CacheVersionedEntryLocalTransactionalSelfTest.class); - suite.addTestSuite(CacheVersionedEntryPartitionedAtomicSelfTest.class); - suite.addTestSuite(CacheVersionedEntryPartitionedTransactionalSelfTest.class); - suite.addTestSuite(CacheVersionedEntryReplicatedAtomicSelfTest.class); - suite.addTestSuite(CacheVersionedEntryReplicatedTransactionalSelfTest.class); + addTestIfNeeded(suite, CacheVersionedEntryLocalAtomicSwapDisabledSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheVersionedEntryLocalTransactionalSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheVersionedEntryPartitionedAtomicSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheVersionedEntryPartitionedTransactionalSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheVersionedEntryReplicatedAtomicSelfTest.class, ignoredTests); + addTestIfNeeded(suite, CacheVersionedEntryReplicatedTransactionalSelfTest.class, ignoredTests); - suite.addTestSuite(GridCacheDhtTxPreloadSelfTest.class); - suite.addTestSuite(GridCacheNearTxPreloadSelfTest.class); - suite.addTestSuite(GridReplicatedTxPreloadTest.class); - suite.addTestSuite(CacheGroupsPreloadTest.class); + addTestIfNeeded(suite, GridCacheDhtTxPreloadSelfTest.class, ignoredTests); + addTestIfNeeded(suite, GridCacheNearTxPreloadSelfTest.class, ignoredTests); + addTestIfNeeded(suite, GridReplicatedTxPreloadTest.class, ignoredTests); + addTestIfNeeded(suite, CacheGroupsPreloadTest.class, ignoredTests); - suite.addTestSuite(IgniteDynamicCacheFilterTest.class); + addTestIfNeeded(suite, IgniteDynamicCacheFilterTest.class, ignoredTests); - suite.addTestSuite(CrossCacheLockTest.class); - suite.addTestSuite(IgniteCrossCacheTxSelfTest.class); + addTestIfNeeded(suite, CrossCacheLockTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCrossCacheTxSelfTest.class, ignoredTests); - suite.addTestSuite(CacheGetFutureHangsSelfTest.class); + addTestIfNeeded(suite, CacheGetFutureHangsSelfTest.class, ignoredTests); - suite.addTestSuite(IgniteCacheSingleGetMessageTest.class); - suite.addTestSuite(IgniteCacheReadFromBackupTest.class); + addTestIfNeeded(suite, IgniteCacheSingleGetMessageTest.class, ignoredTests); + addTestIfNeeded(suite, IgniteCacheReadFromBackupTest.class, ignoredTests); - suite.addTestSuite(MarshallerCacheJobRunNodeRestartTest.class); + addTestIfNeeded(suite, MarshallerCacheJobRunNodeRestartTest.class, ignoredTests); - suite.addTestSuite(IgniteCacheNearOnlyTxTest.class); + addTestIfNeeded(suite, IgniteCacheNearOnlyTxTest.class, ignoredTests); - suite.addTestSuite(IgniteCacheContainsKeyAtomicTest.class); + addTestIfNeeded(suite, IgniteCacheContainsKeyAtomicTest.class, ignoredTests); - suite.addTestSuite(CacheResultIsNotNullOnPartitionLossTest.class); + addTestIfNeeded(suite, CacheResultIsNotNullOnPartitionLossTest.class, ignoredTests); return suite; } From a80c4c8d9d30b4b13c92b500b1e7cb51a0279573 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Wed, 5 Dec 2018 16:11:45 +0300 Subject: [PATCH 277/403] IGNITE-10410: MVCC: Create "Cache 7" test suite for MVCC mode. This closes #5549. --- .../cache/CacheGroupMetricsMBeanTest.java | 20 ++++- .../cache/CacheMetricsManageTest.java | 11 ++- ...amicCacheStartFailWithPersistenceTest.java | 4 + .../cache/MvccCacheGroupMetricsMBeanTest.java | 29 +++++++ .../cache/WalModeChangeAbstractSelfTest.java | 3 + .../cache/WalModeChangeAdvancedSelfTest.java | 4 + .../CacheDataLossOnPartitionMoveTest.java | 6 +- .../CachePageWriteLockUnlockTest.java | 6 +- ...eRebalancingPartitionCountersMvccTest.java | 29 +++++++ ...CacheRebalancingPartitionCountersTest.java | 9 +++ ...eRebalancingWithAsyncClearingMvccTest.java | 36 +++++++++ ...CacheRebalancingWithAsyncClearingTest.java | 13 +++- .../PageEvictionMultinodeAbstractTest.java | 4 + ...nuousQueryAsyncFailoverMvccTxSelfTest.java | 5 ++ ...ContinuousQueryFailoverMvccTxSelfTest.java | 5 ++ ...tegrityWithPrimaryIndexCorruptionTest.java | 9 +++ .../TxRollbackAsyncNearCacheTest.java | 9 +++ .../transactions/TxRollbackAsyncTest.java | 43 +++++++---- ...thSmallTimeoutAndContentionOneKeyTest.java | 10 +++ .../testsuites/IgniteCacheMvccTestSuite7.java | 76 +++++++++++++++++++ .../testsuites/IgniteCacheTestSuite4.java | 2 +- .../testsuites/IgniteCacheTestSuite7.java | 58 +++++++------- .../testsuites/IgnitePdsTestSuite4.java | 3 + 23 files changed, 337 insertions(+), 57 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MvccCacheGroupMetricsMBeanTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersMvccTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingMvccTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java index f8108eb7123a2..3cd730fbe09b7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.affinity.AffinityFunction; import org.apache.ignite.cache.affinity.AffinityFunctionContext; @@ -132,21 +133,25 @@ private static class RoundRobinVariableSizeAffinityFunction implements AffinityF .setGroupName("group1") .setCacheMode(CacheMode.PARTITIONED) .setBackups(3) - .setAffinity(new RoundRobinVariableSizeAffinityFunction()); + .setAffinity(new RoundRobinVariableSizeAffinityFunction()) + .setAtomicityMode(atomicityMode()); CacheConfiguration cCfg2 = new CacheConfiguration() .setName("cache2") .setGroupName("group2") - .setCacheMode(CacheMode.REPLICATED); + .setCacheMode(CacheMode.REPLICATED) + .setAtomicityMode(atomicityMode()); CacheConfiguration cCfg3 = new CacheConfiguration() .setName("cache3") .setGroupName("group2") - .setCacheMode(CacheMode.REPLICATED); + .setCacheMode(CacheMode.REPLICATED) + .setAtomicityMode(atomicityMode()); CacheConfiguration cCfg4 = new CacheConfiguration() .setName("cache4") - .setCacheMode(CacheMode.PARTITIONED); + .setCacheMode(CacheMode.PARTITIONED) + .setAtomicityMode(atomicityMode()); cfg.setCacheConfiguration(cCfg1, cCfg2, cCfg3, cCfg4); @@ -163,6 +168,13 @@ private static class RoundRobinVariableSizeAffinityFunction implements AffinityF return cfg; } + /** + * @return Cache atomicity mode. + */ + protected CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.ATOMIC; + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java index ae00ac9659225..94830be2ac2a8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsManageTest.java @@ -52,6 +52,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -80,6 +81,9 @@ public class CacheMetricsManageTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testJmxNoPdsStatisticsEnable() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9224"); + testJmxStatisticsEnable(false); } @@ -87,6 +91,9 @@ public void testJmxNoPdsStatisticsEnable() throws Exception { * @throws Exception If failed. */ public void testJmxPdsStatisticsEnable() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10421"); + testJmxStatisticsEnable(true); } @@ -101,7 +108,7 @@ public void testCacheManagerStatisticsEnable() throws Exception { .setName(CACHE1) .setGroupName(GROUP) .setCacheMode(CacheMode.PARTITIONED) - .setAtomicityMode(CacheAtomicityMode.ATOMIC); + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); mgr1.createCache(CACHE1, cfg1); @@ -490,7 +497,7 @@ private CacheMetricsMXBean mxBean(int nodeIdx, String cacheName, Class() { @Override public void applyx(Ignite ignite) throws IgniteCheckedException { createCache(ignite, cacheConfig(LOCAL).setDataRegionName(REGION_VOLATILE)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java index be0f5df158761..004ad58b87c7d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.IgniteClientReconnectAbstractTest; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -133,6 +134,9 @@ public void testCacheCleanup() throws Exception { * @throws Exception If failed. */ public void testJoin() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10421"); + checkJoin(false); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java index 9ff072500b6fc..acbfa8cedd672 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java @@ -36,13 +36,14 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.cache.GridCacheUtils; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.testframework.GridTestNode; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -119,6 +120,9 @@ private String grp(int idx) { * @throws Exception if failed. */ public void testDataLossOnPartitionMove() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10421"); + try { Ignite ignite = startGridsMultiThreaded(GRIDS_CNT / 2, false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java index 41c588268a8f2..5b2476140052e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java @@ -21,6 +21,7 @@ import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -51,8 +52,9 @@ public class CachePageWriteLockUnlockTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME). - setAffinity(new RendezvousAffinityFunction(false, 32))); + cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); cfg.setActiveOnStart(false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersMvccTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersMvccTest.java new file mode 100644 index 0000000000000..4166cfcabb5df --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersMvccTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.distributed.rebalancing; + +import org.apache.ignite.cache.CacheAtomicityMode; + +/** + * + */ +public class GridCacheRebalancingPartitionCountersMvccTest extends GridCacheRebalancingPartitionCountersTest { + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java index 258e36e531322..6523eb9403bf2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -61,6 +62,7 @@ public class GridCacheRebalancingPartitionCountersTest extends GridCommonAbstrac .setMaxSize(100L * 1024 * 1024)) .setWalMode(WALMode.LOG_ONLY)) .setCacheConfiguration(new CacheConfiguration(CACHE_NAME) + .setAtomicityMode(atomicityMode()) .setBackups(2) .setRebalanceBatchSize(4096) // Force to create several supply messages during rebalancing. .setAffinity( @@ -79,6 +81,13 @@ public class GridCacheRebalancingPartitionCountersTest extends GridCommonAbstrac U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); } + /** + * @return Cache atomicity mode. + */ + protected CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.ATOMIC; + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingMvccTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingMvccTest.java new file mode 100644 index 0000000000000..a61d6d00c5928 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingMvccTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.distributed.rebalancing; + +import org.apache.ignite.cache.CacheAtomicityMode; + +/** + * + */ +public class GridCacheRebalancingWithAsyncClearingMvccTest extends GridCacheRebalancingWithAsyncClearingTest { + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10421"); + + super.setUp(); + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java index 1b176ae8a07ba..b2c8712a2a02b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.PartitionLossPolicy; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; @@ -31,9 +32,9 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Assert; @@ -63,7 +64,8 @@ public class GridCacheRebalancingWithAsyncClearingTest extends GridCommonAbstrac .setMaxSize(100L * 1024 * 1024)) ); - cfg.setCacheConfiguration(new CacheConfiguration(CACHE_NAME) + cfg.setCacheConfiguration(new CacheConfiguration<>(CACHE_NAME) + .setAtomicityMode(atomicityMode()) .setBackups(2) .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) .setIndexedTypes(Integer.class, Integer.class) @@ -92,6 +94,13 @@ public class GridCacheRebalancingWithAsyncClearingTest extends GridCommonAbstrac cleanPersistenceDir(); } + /** + * @return Atomicity mode. + */ + protected CacheAtomicityMode atomicityMode() { + return CacheAtomicityMode.ATOMIC; + } + /** * Test that partition clearing doesn't block partitions map exchange. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java index 777c2d7e999cc..e30f1cf040adf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * @@ -46,6 +47,9 @@ public abstract class PageEvictionMultinodeAbstractTest extends PageEvictionAbst /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + startGridsMultiThreaded(4, false); clientGrid = startGrid("client"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java index 334d2190c998b..7c15f38a7e14e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryAsyncFailoverMvccTxSelfTest.java @@ -50,4 +50,9 @@ public class CacheContinuousQueryAsyncFailoverMvccTxSelfTest extends CacheConti @Override public void testLeftPrimaryAndBackupNodes() throws Exception { fail("https://issues.apache.org/jira/browse/IGNITE-10047"); } + + /** {@inheritDoc} */ + @Override public void testFailoverStartStopBackup() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java index 76d1689715562..2d578f28e3d84 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverMvccTxSelfTest.java @@ -45,4 +45,9 @@ public class CacheContinuousQueryFailoverMvccTxSelfTest extends CacheContinuousQ @Override public void testLeftPrimaryAndBackupNodes() throws Exception { fail("https://issues.apache.org/jira/browse/IGNITE-10047"); } + + /** {@inheritDoc} */ + @Override public void testFailoverStartStopBackup() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10391"); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java index fd31ee5e78f56..616bf97a72059 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler; import org.apache.ignite.internal.processors.cache.tree.SearchRow; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; @@ -44,6 +45,14 @@ public class TransactionIntegrityWithPrimaryIndexCorruptionTest extends Abstract /** Corruption enabled flag. */ private static volatile boolean corruptionEnabled; + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10470"); + + super.beforeTest(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { corruptionEnabled = false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java index 5caa1a06325b8..dc115e1dc62d6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.cache.transactions; +import org.apache.ignite.testframework.MvccFeatureChecker; + /** * Tests an ability to async rollback near transactions. */ @@ -25,4 +27,11 @@ public class TxRollbackAsyncNearCacheTest extends TxRollbackAsyncTest { @Override protected boolean nearCacheEnabled() { return true; } + + /** {@inheritDoc} */ + @Override public void setUp() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.setUp(); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java index 4a074e6f1225a..d742ed504655c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java @@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistRequest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.internal.util.future.GridFutureAdapter; @@ -97,7 +98,6 @@ import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; -import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK; @@ -124,14 +124,6 @@ public class TxRollbackAsyncTest extends GridCommonAbstractTest { /** */ public static final String LABEL = "wLockTx"; - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10410"); - - super.beforeTestsStarted(); - } - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -221,6 +213,9 @@ private Ignite startClient() throws Exception { * */ public void testRollbackSimple() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + startClient(); for (Ignite ignite : G.allGrids()) { @@ -235,7 +230,7 @@ public void testRollbackSimple() throws Exception { */ private void testRollbackSimple0(Ignite near) throws Exception { // Normal rollback after put. - Transaction tx = near.transactions().txStart(PESSIMISTIC, READ_COMMITTED); + Transaction tx = near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ); near.cache(CACHE_NAME).put(0, 0); @@ -399,7 +394,7 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN txReadyFut.onDone(tx); // Will block on lock request until rolled back asynchronously. - Object o = tryLockNode.cache(CACHE_NAME).get(0); + Object o = tryLockNode.cache(CACHE_NAME).getAndPut(0, 0); assertNull(o); // If rolled back by close, previous get will return null. } @@ -489,6 +484,9 @@ public void testEnlistManyWrite() throws Exception { * */ public void testEnlistManyReadOptimistic() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + return; // Optimistic transactions are not supported by MVCC. + testEnlistMany(false, SERIALIZABLE, OPTIMISTIC); } @@ -496,6 +494,9 @@ public void testEnlistManyReadOptimistic() throws Exception { * */ public void testEnlistManyWriteOptimistic() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + return; // Optimistic transactions are not supported by MVCC. + testEnlistMany(true, SERIALIZABLE, OPTIMISTIC); } @@ -540,13 +541,20 @@ private void testEnlistMany(boolean write, TransactionIsolation isolation, * Rollback tx while near lock request is delayed. */ public void testRollbackDelayNearLockRequest() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-9470"); + final Ignite client = startClient(); final Ignite prim = primaryNode(0, CACHE_NAME); final TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi)client.configuration().getCommunicationSpi(); - spi.blockMessages(GridNearLockRequest.class, prim.name()); + boolean mvcc = MvccFeatureChecker.forcedMvcc(); + + Class msgCls = mvcc ? GridNearTxEnlistRequest.class : GridNearLockRequest.class; + + spi.blockMessages(msgCls, prim.name()); final IgniteInternalFuture rollbackFut = runAsync(new Callable() { @Override public Void call() throws Exception { @@ -564,7 +572,7 @@ public void testRollbackDelayNearLockRequest() throws Exception { fail(); } catch (CacheException e) { - assertTrue(X.hasCause(e, TransactionRollbackException.class)); + assertTrue(X.getFullStackTrace(e),X.hasCause(e, TransactionRollbackException.class)); } rollbackFut.get(); @@ -639,6 +647,9 @@ public void testRollbackDelayFinishRequest() throws Exception { * */ public void testMixedAsyncRollbackTypes() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10434"); + final Ignite client = startClient(); final AtomicBoolean stop = new AtomicBoolean(); @@ -669,6 +680,8 @@ public void testMixedAsyncRollbackTypes() throws Exception { for (Ignite ignite : G.allGrids()) perNodeTxs.put(ignite, new ArrayBlockingQueue<>(1000)); + boolean mvcc = MvccFeatureChecker.forcedMvcc(); + IgniteInternalFuture txFut = multithreadedAsync(() -> { while (!stop.get()) { int nodeId = r.nextInt(GRID_CNT + 1); @@ -676,8 +689,8 @@ public void testMixedAsyncRollbackTypes() throws Exception { // Choose random node to start tx on. Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId); - TransactionConcurrency conc = TC_VALS[r.nextInt(TC_VALS.length)]; - TransactionIsolation isolation = TI_VALS[r.nextInt(TI_VALS.length)]; + TransactionConcurrency conc = mvcc ? PESSIMISTIC : TC_VALS[r.nextInt(TC_VALS.length)]; + TransactionIsolation isolation = mvcc ? REPEATABLE_READ :TI_VALS[r.nextInt(TI_VALS.length)]; // Timeout is necessary otherwise deadlock is possible due to randomness of lock acquisition. long timeout = r.nextInt(50) + 50; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java index 62f46352391bf..cac0427b4cee2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -112,6 +113,9 @@ public class TxWithSmallTimeoutAndContentionOneKeyTest extends GridCommonAbstrac * @return Random transaction type. */ protected TransactionConcurrency transactionConcurrency() { + if (MvccFeatureChecker.forcedMvcc()) + return PESSIMISTIC; + ThreadLocalRandom random = ThreadLocalRandom.current(); return random.nextBoolean() ? OPTIMISTIC : PESSIMISTIC; @@ -121,6 +125,9 @@ protected TransactionConcurrency transactionConcurrency() { * @return Random transaction isolation level. */ protected TransactionIsolation transactionIsolation(){ + if (MvccFeatureChecker.forcedMvcc()) + return REPEATABLE_READ; + ThreadLocalRandom random = ThreadLocalRandom.current(); switch (random.nextInt(3)) { @@ -150,6 +157,9 @@ protected long randomTimeOut() { * @throws Exception If failed. */ public void test() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10455"); + startGrids(4); client = true; diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java new file mode 100644 index 0000000000000..403b5a026f05a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.processors.authentication.Authentication1kUsersNodeRestartTest; +import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest; +import org.apache.ignite.internal.processors.authentication.AuthenticationOnNotActiveClusterTest; +import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNPEOnStartTest; +import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNodeRestartTest; +import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest; +import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest; +import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest; +import org.apache.ignite.internal.processors.cache.MvccCacheGroupMetricsMBeanTest; +import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest; +import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersMvccTest; +import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest; +import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest; + +/** + * + */ +public class IgniteCacheMvccTestSuite7 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + HashSet ignoredTests = new HashSet<>(128); + + // Other non-tx tests. + ignoredTests.add(CheckpointBufferDeadlockTest.class);// + ignoredTests.add(AuthenticationConfigurationClusterTest.class);// + ignoredTests.add(AuthenticationProcessorSelfTest.class); + ignoredTests.add(AuthenticationOnNotActiveClusterTest.class); + ignoredTests.add(AuthenticationProcessorNodeRestartTest.class); + ignoredTests.add(AuthenticationProcessorNPEOnStartTest.class); + ignoredTests.add(Authentication1kUsersNodeRestartTest.class); + ignoredTests.add(CacheDataRegionConfigurationTest.class); + ignoredTests.add(Cache64kPartitionsTest.class); + + // Skip classes which Mvcc implementations are added in this method below. + // TODO IGNITE-10175: refactor these tests (use assume) to support both mvcc and non-mvcc modes after moving to JUnit4/5. + ignoredTests.add(CacheGroupMetricsMBeanTest.class); // See MvccCacheGroupMetricsMBeanTest + ignoredTests.add(GridCacheRebalancingPartitionCountersTest.class); // See GridCacheRebalancingPartitionCountersMvccTest + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 7"); + + suite.addTest(IgniteCacheTestSuite7.suite(ignoredTests)); + + // Add Mvcc clones. + suite.addTestSuite(MvccCacheGroupMetricsMBeanTest.class); + suite.addTestSuite(GridCacheRebalancingPartitionCountersMvccTest.class); + + + return suite; + } + +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java index 04620c8f7fed1..24c4e594b4bdf 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http:www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java index 6c48ecc2da6f7..f7e102019ce83 100755 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java @@ -33,9 +33,8 @@ import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest; import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest; -import org.apache.ignite.internal.processors.cache.distributed.CachePageWriteLockUnlockTest; -import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest; import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest; import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest; @@ -45,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.transactions.TransactionIntegrityWithPrimaryIndexCorruptionTest; import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest; import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -63,46 +63,44 @@ public static TestSuite suite() throws Exception { * @return Test suite. * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite(Set ignoredTests) throws Exception { + public static TestSuite suite(Set ignoredTests) { TestSuite suite = new TestSuite("IgniteCache With Persistence Test Suite"); - suite.addTestSuite(CheckpointBufferDeadlockTest.class); - suite.addTestSuite(IgniteCacheStartWithLoadTest.class); - - suite.addTestSuite(AuthenticationConfigurationClusterTest.class); - suite.addTestSuite(AuthenticationProcessorSelfTest.class); - suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class); - suite.addTestSuite(AuthenticationProcessorNodeRestartTest.class); - suite.addTestSuite(AuthenticationProcessorNPEOnStartTest.class); - suite.addTestSuite(Authentication1kUsersNodeRestartTest.class); + GridTestUtils.addTestIfNeeded(suite, CheckpointBufferDeadlockTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteCacheStartWithLoadTest.class, ignoredTests); - suite.addTestSuite(CacheDataRegionConfigurationTest.class); + GridTestUtils.addTestIfNeeded(suite, AuthenticationConfigurationClusterTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, AuthenticationProcessorSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, AuthenticationOnNotActiveClusterTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, AuthenticationProcessorNodeRestartTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, AuthenticationProcessorNPEOnStartTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Authentication1kUsersNodeRestartTest.class, ignoredTests); - suite.addTestSuite(WalModeChangeAdvancedSelfTest.class); - suite.addTestSuite(WalModeChangeSelfTest.class); - suite.addTestSuite(WalModeChangeCoordinatorNotAffinityNodeSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheDataRegionConfigurationTest.class, ignoredTests); - suite.addTestSuite(Cache64kPartitionsTest.class); - suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class); - suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class); + GridTestUtils.addTestIfNeeded(suite, WalModeChangeAdvancedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, WalModeChangeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, WalModeChangeCoordinatorNotAffinityNodeSelfTest.class, ignoredTests); - suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class); - suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class); + GridTestUtils.addTestIfNeeded(suite, Cache64kPartitionsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingPartitionCountersTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingWithAsyncClearingTest.class, ignoredTests); - suite.addTestSuite(CacheGroupMetricsMBeanTest.class); - suite.addTestSuite(CacheMetricsManageTest.class); - suite.addTestSuite(PageEvictionMultinodeMixedRegionsTest.class); + GridTestUtils.addTestIfNeeded(suite, IgnitePdsCacheAssignmentNodeRestartsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, TxRollbackAsyncWithPersistenceTest.class, ignoredTests); - suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheGroupMetricsMBeanTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheMetricsManageTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionMultinodeMixedRegionsTest.class, ignoredTests); - suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class); + GridTestUtils.addTestIfNeeded(suite, IgniteDynamicCacheStartFailWithPersistenceTest.class, ignoredTests); - suite.addTestSuite(CacheRentingStateRepairTest.class); + GridTestUtils.addTestIfNeeded(suite, TxWithSmallTimeoutAndContentionOneKeyTest.class, ignoredTests); - suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class); - suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheRentingStateRepairTest.class, ignoredTests); - suite.addTestSuite(CachePageWriteLockUnlockTest.class); + GridTestUtils.addTestIfNeeded(suite, TransactionIntegrityWithPrimaryIndexCorruptionTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheDataLossOnPartitionMoveTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java index 5e6d45a1fb8ed..c6fd2e2354b2c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java @@ -21,6 +21,7 @@ import org.apache.ignite.cache.ResetLostPartitionTest; import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.IgniteRebalanceOnCachesStoppingOrDestroyingTest; +import org.apache.ignite.internal.processors.cache.distributed.CachePageWriteLockUnlockTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest; @@ -54,6 +55,8 @@ public static TestSuite suite() { suite.addTestSuite(IgniteRebalanceOnCachesStoppingOrDestroyingTest.class); + suite.addTestSuite(CachePageWriteLockUnlockTest.class); + suite.addTestSuite(IgnitePdsCacheWalDisabledOnRebalancingTest.class); return suite; From 856aa595951e7d3c335e2059b790f6f663066023 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Mon, 10 Dec 2018 14:42:25 +0300 Subject: [PATCH 278/403] IGNITE-10367: MVCC: Create "Cache 8" test suite for MVCC mode. This closes #5585. --- .../TransactionMetricsMxBeanImplTest.java | 14 +- .../CacheGroupsMetricsRebalanceTest.java | 4 +- .../cache/CacheMetricsEntitiesCountTest.java | 47 ++++--- .../CacheMetricsForClusterGroupSelfTest.java | 8 ++ .../cache/CacheValidatorMetricsTest.java | 3 + .../GridCacheAbstractIteratorsSelfTest.java | 4 + .../GridCachePreloadingEvictionsSelfTest.java | 3 + .../cache/GridEvictionPolicyMBeansTest.java | 32 +++-- ...ogyValidatorAbstractTxCacheGroupsTest.java | 63 ++++++--- ...eTopologyValidatorAbstractTxCacheTest.java | 36 +++-- ...teTopologyValidatorGridSplitCacheTest.java | 4 + ...datorNearPartitionedTxCacheGroupsTest.java | 8 ++ ...gyValidatorNearPartitionedTxCacheTest.java | 8 ++ ...apCacheMetricsForClusterGroupSelfTest.java | 8 +- .../GridCacheMvccNearEvictionSelfTest.java | 39 ++++++ .../near/GridCacheNearMetricsSelfTest.java | 3 + ...CachePartitionedHitsAndMissesSelfTest.java | 3 + .../GridCachePartitionedMetricsSelfTest.java | 8 ++ .../GridCacheRebalancingCancelTest.java | 2 + ...GridCacheRebalancingSyncCheckDataTest.java | 2 + .../GridCacheRebalancingSyncSelfTest.java | 13 ++ ...ebalancingUnmarshallingFailedSelfTest.java | 2 + .../GridCacheReplicatedMetricsSelfTest.java | 8 ++ .../eviction/DhtAndNearEvictionTest.java | 13 ++ ...ConcurrentEvictionConsistencySelfTest.java | 35 +++-- .../GridCacheConcurrentEvictionsSelfTest.java | 19 ++- ...GridCacheEmptyEntriesAbstractSelfTest.java | 4 + .../GridCacheEmptyEntriesLocalSelfTest.java | 10 ++ ...dCacheEmptyEntriesPartitionedSelfTest.java | 8 ++ .../GridCacheEvictionFilterSelfTest.java | 5 + .../GridCacheEvictionLockUnlockSelfTest.java | 10 ++ .../GridCacheEvictionTouchSelfTest.java | 8 ++ .../lru/LruNearEvictionPolicySelfTest.java | 13 ++ ...LruNearOnlyNearEvictionPolicySelfTest.java | 27 ++++ .../paged/PageEvictionDataStreamerTest.java | 8 +- .../paged/PageEvictionMetricTest.java | 18 ++- .../PageEvictionMultinodeAbstractTest.java | 38 +++-- ...PageEvictionMultinodeMixedRegionsTest.java | 2 +- ...eEvictionPagesRecyclingAndReusingTest.java | 29 ++++ .../paged/PageEvictionReadThroughTest.java | 31 +++++ .../paged/PageEvictionTouchOrderTest.java | 30 ++++ ...PageEvictionWithRebalanceAbstractTest.java | 18 ++- .../GridCacheLocalIteratorsSelfTest.java | 8 ++ .../local/GridCacheLocalMetricsSelfTest.java | 10 ++ ...itePdsCacheAssignmentNodeRestartsTest.java | 4 + .../IgniteCacheEvictionSelfTestSuite.java | 70 +++++----- .../IgniteCacheIteratorsSelfTestSuite.java | 12 +- .../IgniteCacheMetricsSelfTestSuite.java | 46 +++--- .../testsuites/IgniteCacheMvccTestSuite7.java | 4 + .../testsuites/IgniteCacheMvccTestSuite8.java | 131 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite8.java | 34 +++-- .../IgniteTopologyValidatorTestSuite.java | 32 +++-- 52 files changed, 815 insertions(+), 184 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheMvccNearEvictionSelfTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite8.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java index 91bd9bcf98271..4c192715345ae 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java @@ -17,13 +17,13 @@ package org.apache.ignite.internal; -import javax.management.MBeanServer; -import javax.management.MBeanServerInvocationHandler; -import javax.management.ObjectName; import java.lang.management.ManagementFactory; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import javax.management.MBeanServer; +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheRebalanceMode; @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -81,6 +82,13 @@ public class TransactionMetricsMxBeanImplTest extends GridCommonAbstractTest { super.afterTest(); } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + + super.beforeTestsStarted(); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java index 3e90625cf2cd3..d06563173c699 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java @@ -97,7 +97,7 @@ public class CacheGroupsMetricsRebalanceTest extends GridCommonAbstractTest { .setName(CACHE1) .setGroupName(GROUP) .setCacheMode(CacheMode.PARTITIONED) - .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setRebalanceMode(CacheRebalanceMode.ASYNC) .setRebalanceBatchSize(100) .setStatisticsEnabled(true); @@ -108,7 +108,7 @@ public class CacheGroupsMetricsRebalanceTest extends GridCommonAbstractTest { CacheConfiguration cfg3 = new CacheConfiguration() .setName(CACHE3) .setCacheMode(CacheMode.PARTITIONED) - .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setRebalanceMode(CacheRebalanceMode.ASYNC) .setRebalanceBatchSize(100) .setStatisticsEnabled(true) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEntitiesCountTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEntitiesCountTest.java index 1ab7c40ced6a0..09a438b4b448d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEntitiesCountTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEntitiesCountTest.java @@ -17,14 +17,19 @@ package org.apache.ignite.internal.processors.cache; +import java.util.ArrayList; +import java.util.Collection; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -48,35 +53,45 @@ public class CacheMetricsEntitiesCountTest extends GridCommonAbstractTest { CachePeekMode.ONHEAP, CachePeekMode.PRIMARY, CachePeekMode.BACKUP, CachePeekMode.NEAR}; /** Cache count. */ - private static final int CACHE_CNT = 4; + private static int cacheCnt = 4; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - CacheConfiguration ccfg0 = new CacheConfiguration<>() - .setName(CACHE_PREFIX + 0) - .setCacheMode(CacheMode.LOCAL); + Collection ccfgs = new ArrayList<>(4); - CacheConfiguration ccfg1 = new CacheConfiguration<>() - .setName(CACHE_PREFIX + 1) + ccfgs.add(new CacheConfiguration<>() + .setName(CACHE_PREFIX + 0) .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) - .setCacheMode(CacheMode.REPLICATED); + .setCacheMode(CacheMode.REPLICATED) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); - CacheConfiguration ccfg2 = new CacheConfiguration<>() - .setName(CACHE_PREFIX + 2) + ccfgs.add(new CacheConfiguration<>() + .setName(CACHE_PREFIX + 1) .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) .setCacheMode(CacheMode.PARTITIONED) - .setBackups(1); + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); - CacheConfiguration ccfg3 = new CacheConfiguration<>() - .setName(CACHE_PREFIX + 3) + ccfgs.add(new CacheConfiguration<>() + .setName(CACHE_PREFIX + 2) .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) .setCacheMode(CacheMode.PARTITIONED) .setBackups(1) - .setNearConfiguration(new NearCacheConfiguration<>()); + .setNearConfiguration(new NearCacheConfiguration<>()) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.LOCAL_CACHE)) { + ccfgs.add(new CacheConfiguration<>() + .setName(CACHE_PREFIX + 3) + .setCacheMode(CacheMode.LOCAL) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + } + + cacheCnt = ccfgs.size(); - cfg.setCacheConfiguration(ccfg0, ccfg1, ccfg2, ccfg3); + cfg.setCacheConfiguration(U.toArray(ccfgs, new CacheConfiguration[cacheCnt])); return cfg; } @@ -93,11 +108,11 @@ public void testEnitiesCount() throws Exception { awaitPartitionMapExchange(); for (int igniteIdx = 0; igniteIdx < GRID_CNT; igniteIdx++) - for (int cacheIdx = 0; cacheIdx < CACHE_CNT; cacheIdx++) + for (int cacheIdx = 0; cacheIdx < cacheCnt; cacheIdx++) fillCache(igniteIdx, cacheIdx); for (int igniteIdx = 0; igniteIdx < GRID_CNT; igniteIdx++) - for (int cacheIdx = 0; cacheIdx < CACHE_CNT; cacheIdx++) + for (int cacheIdx = 0; cacheIdx < cacheCnt; cacheIdx++) checkCache(igniteIdx, cacheIdx); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java index df60c42e8d4c6..9616a24fbed76 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; @@ -63,6 +64,13 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest /** Daemon grid. */ private boolean daemon; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java index 4a950dda8f1a1..b856c8840c6c4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.ignite.Ignite; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.PartitionLossPolicy; import org.apache.ignite.cluster.ClusterNode; @@ -48,11 +49,13 @@ public class CacheValidatorMetricsTest extends GridCommonAbstractTest implements .setName(CACHE_NAME_1) .setCacheMode(CacheMode.PARTITIONED) .setBackups(0) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_ALL); CacheConfiguration cCfg2 = new CacheConfiguration() .setName(CACHE_NAME_2) .setCacheMode(CacheMode.REPLICATED) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setTopologyValidator(new TopologyValidator() { @Override public boolean validate(Collection nodes) { return nodes.size() == 2; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractIteratorsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractIteratorsSelfTest.java index 85e3d6544298c..570f43c9f6609 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractIteratorsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractIteratorsSelfTest.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.util.typedef.CA; import org.apache.ignite.internal.util.typedef.CAX; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Tests for cache iterators. @@ -96,6 +97,9 @@ public void testCacheIteratorMultithreaded() throws Exception { * @throws Exception If failed. */ public void testEntrySetIterator() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10082"); + assert jcache().localSize(CachePeekMode.ALL) == entryCount(); int cnt = 0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java index 2a10d9723641c..6b3a8635c5580 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCachePreloadingEvictionsSelfTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; @@ -102,6 +103,8 @@ public class GridCachePreloadingEvictionsSelfTest extends GridCommonAbstractTest */ public void testEvictions() throws Exception { try { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + final Ignite ignite1 = startGrid(1); final IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridEvictionPolicyMBeansTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridEvictionPolicyMBeansTest.java index b093bb2f33307..cb746668a9591 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridEvictionPolicyMBeansTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridEvictionPolicyMBeansTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -36,6 +37,13 @@ public GridEvictionPolicyMBeansTest() { super(true); } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + + super.beforeTestsStarted(); + } + /** * {@inheritDoc} * @@ -60,8 +68,6 @@ public GridEvictionPolicyMBeansTest() { ncf.setNearEvictionPolicyFactory(new LruEvictionPolicyFactory<>(40, 10, 500)); - cache1.setNearConfiguration(ncf); - CacheConfiguration cache2 = defaultCacheConfiguration(); cache2.setName("cache2"); @@ -81,7 +87,11 @@ public GridEvictionPolicyMBeansTest() { lep.setMaxMemorySize(500); lep.setMaxSize(40); ncf.setNearEvictionPolicy(lep); - cache2.setNearConfiguration(ncf); + + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) { + cache1.setNearConfiguration(ncf); + cache2.setNearConfiguration(ncf); + } cfg.setCacheConfiguration(cache1, cache2); @@ -94,17 +104,21 @@ public void testEvictionPolicyBeans() throws Exception{ checkBean("cache1", "org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy", "BatchSize", 10); checkBean("cache1", "org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy", "MaxMemorySize", 20L); - checkBean("cache1-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxSize", 40); - checkBean("cache1-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "BatchSize", 10); - checkBean("cache1-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxMemorySize", 500L); + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) { + checkBean("cache1-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxSize", 40); + checkBean("cache1-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "BatchSize", 10); + checkBean("cache1-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxMemorySize", 500L); + } checkBean("cache2", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxSize", 30); checkBean("cache2", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "BatchSize", 10); checkBean("cache2", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxMemorySize", 125L); - checkBean("cache2-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxSize", 40); - checkBean("cache2-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "BatchSize", 10); - checkBean("cache2-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxMemorySize", 500L); + if (!MvccFeatureChecker.forcedMvcc() || MvccFeatureChecker.isSupported(MvccFeatureChecker.Feature.NEAR_CACHE)) { + checkBean("cache2-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxSize", 40); + checkBean("cache2-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "BatchSize", 10); + checkBean("cache2-near", "org.apache.ignite.cache.eviction.lru.LruEvictionPolicy", "MaxMemorySize", 500L); + } } /** Checks that a bean with the specified group and name is available and has the expected attribute */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheGroupsTest.java index 07875350886e8..574a4deaf3ae9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheGroupsTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -43,10 +44,20 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheGroupsTest /** {@inheritDoc} */ @Override public void testTopologyValidator() throws Exception { - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(CACHE_NAME_1); - putValid(CACHE_NAME_3); - commitFailed(tx); + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + putInvalid(CACHE_NAME_1); + } + + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + putInvalid(CACHE_NAME_3); + } + + if (!MvccFeatureChecker.forcedMvcc()) { + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + putValid(CACHE_NAME_3); + commitFailed(tx); + } } assertEmpty(CACHE_NAME_1); // Rolled back. @@ -69,10 +80,19 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheGroupsTest startGrid(1); - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(CACHE_NAME_1); - putValid(CACHE_NAME_3); - tx.commit(); + if (!MvccFeatureChecker.forcedMvcc()) { + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + putValid(CACHE_NAME_3); + tx.commit(); + } + } + else { + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + putValid(CACHE_NAME_3); + tx.commit(); + } } remove(CACHE_NAME_1); @@ -96,19 +116,30 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheGroupsTest assertEmpty(CACHE_NAME_3); // Rolled back. - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(CACHE_NAME_1); - putValid(CACHE_NAME_3); - commitFailed(tx); + if (!MvccFeatureChecker.forcedMvcc()) { + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + putValid(CACHE_NAME_3); + commitFailed(tx); + } } assertEmpty(CACHE_NAME_1); // Rolled back. assertEmpty(CACHE_NAME_3); // Rolled back. - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(DEFAULT_CACHE_NAME); - putValid(CACHE_NAME_3); - tx.commit(); + if (!MvccFeatureChecker.forcedMvcc()) { + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(DEFAULT_CACHE_NAME); + putValid(CACHE_NAME_3); + tx.commit(); + } + } + else { + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { + putValid(DEFAULT_CACHE_NAME); + putValid(CACHE_NAME_3); + tx.commit(); + } } remove(DEFAULT_CACHE_NAME); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java index 781d88de40134..817fda37cd523 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractTxCacheTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -42,20 +43,22 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheTest extends IgniteT /** {@inheritDoc} */ @Override public void testTopologyValidator() throws Exception { - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(CACHE_NAME_1); - commitFailed(tx); - } - try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { putInvalid(CACHE_NAME_1); } - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(CACHE_NAME_1); - putValid(DEFAULT_CACHE_NAME); - putValid(CACHE_NAME_2); - commitFailed(tx); + if (!MvccFeatureChecker.forcedMvcc()) { + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + commitFailed(tx); + } + + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + putValid(DEFAULT_CACHE_NAME); + putValid(CACHE_NAME_2); + commitFailed(tx); + } } assertEmpty(DEFAULT_CACHE_NAME); // rolled back @@ -72,7 +75,7 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheTest extends IgniteT startGrid(1); - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { putValid(CACHE_NAME_1); tx.commit(); } @@ -96,16 +99,19 @@ public abstract class IgniteTopologyValidatorAbstractTxCacheTest extends IgniteT assertEmpty(DEFAULT_CACHE_NAME); // rolled back assertEmpty(CACHE_NAME_1); // rolled back - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - putValid(CACHE_NAME_1); - commitFailed(tx); + if (!MvccFeatureChecker.forcedMvcc()) { + try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + putValid(CACHE_NAME_1); + commitFailed(tx); + } } + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { putInvalid(CACHE_NAME_1); } - try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { putValid(DEFAULT_CACHE_NAME); putValid(CACHE_NAME_2); tx.commit(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorGridSplitCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorGridSplitCacheTest.java index c315ba50f4707..60a4b6d93309e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorGridSplitCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorGridSplitCacheTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; @@ -197,6 +198,9 @@ private String testCacheName(int idx) { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-7952"); + super.beforeTest(); startGridsMultiThreaded(GRID_CNT); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest.java index acfad80fc5a9d..d9cf301885882 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Topology validator test. @@ -28,4 +29,11 @@ public class IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest extends @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheTest.java index 800c2e654d333..c1a8fa81898eb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorNearPartitionedTxCacheTest.java @@ -18,11 +18,19 @@ package org.apache.ignite.internal.processors.cache; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Topology validator test */ public class IgniteTopologyValidatorNearPartitionedTxCacheTest extends IgniteTopologyValidatorPartitionedTxCacheTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected NearCacheConfiguration nearConfiguration() { return new NearCacheConfiguration(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java index 9ff6db7c2a5ee..6adb9d687595f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java @@ -17,14 +17,15 @@ package org.apache.ignite.internal.processors.cache; +import java.util.concurrent.CountDownLatch; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.Event; import org.apache.ignite.events.EventType; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import java.util.concurrent.CountDownLatch; import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; @@ -140,11 +141,16 @@ private void assertGetOffHeapPrimaryEntriesCount(String cacheName, int count) th } } + /** + * @param cacheName Cache name. + * @return Cache configuration. + */ private static CacheConfiguration cacheConfiguration(String cacheName) { CacheConfiguration cfg = new CacheConfiguration<>(cacheName); cfg.setBackups(1); cfg.setStatisticsEnabled(true); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); return cfg; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheMvccNearEvictionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheMvccNearEvictionSelfTest.java new file mode 100644 index 0000000000000..7940a9efd5ba1 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheMvccNearEvictionSelfTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.distributed.near; + +import org.apache.ignite.cache.CacheAtomicityMode; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; + +/** + * Test for mvcc cache. + */ +public class GridCacheMvccNearEvictionSelfTest extends GridCacheNearEvictionSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + super.beforeTestsStarted(); + } + + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL_SNAPSHOT; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java index 70b06f34f2d9f..97391a35895e1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -83,6 +84,8 @@ protected int keyCount() { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + super.beforeTest(); for (int i = 0; i < gridCount(); i++) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedHitsAndMissesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedHitsAndMissesSelfTest.java index 64f594084ea7e..cd048c3320d82 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedHitsAndMissesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedHitsAndMissesSelfTest.java @@ -32,6 +32,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.stream.StreamReceiver; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -99,6 +100,8 @@ protected CacheConfiguration cacheConfiguration() throws Exception { * @throws Exception If failed. */ public void testHitsAndMisses() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + startGrids(GRID_CNT); awaitPartitionMapExchange(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMetricsSelfTest.java index a67e01f4e667e..e946c435a1619 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMetricsSelfTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheTransactionalAbstractMetricsSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; @@ -32,6 +33,13 @@ public class GridCachePartitionedMetricsSelfTest extends GridCacheTransactionalA /** */ private static final int GRID_CNT = 2; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java index 3965290480a3d..d61b9bb6bb843 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.distributed.rebalancing; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; @@ -71,6 +72,7 @@ public void testClientNodeJoinAtRebalancing() throws Exception { .setRebalanceMode(CacheRebalanceMode.ASYNC) .setBackups(1) .setRebalanceOrder(2) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setAffinity(new RendezvousAffinityFunction(false))); for (int i = 0; i < 2048; i++) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncCheckDataTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncCheckDataTest.java index 8a43b65d75a70..ccadfbd0b1eba 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncCheckDataTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncCheckDataTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -48,6 +49,7 @@ public class GridCacheRebalancingSyncCheckDataTest extends GridCommonAbstractTes CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); ccfg.setCacheMode(REPLICATED); ccfg.setRebalanceMode(SYNC); + ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); cfg.setCacheConfiguration(ccfg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java index 0bb35d1fb7526..7f5917981d77d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java @@ -24,6 +24,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cluster.ClusterNode; @@ -54,6 +55,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -125,6 +127,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest { cachePCfg.setRebalanceBatchSize(1); cachePCfg.setRebalanceBatchesPrefetchCount(1); cachePCfg.setRebalanceOrder(2); + cachePCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); CacheConfiguration cachePCfg2 = new CacheConfiguration<>(DEFAULT_CACHE_NAME); @@ -134,6 +137,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest { cachePCfg2.setBackups(1); cachePCfg2.setRebalanceOrder(2); cachePCfg2.setRebalanceDelay(5000); + cachePCfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); CacheConfiguration cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); @@ -142,6 +146,8 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest { cacheRCfg.setRebalanceMode(CacheRebalanceMode.SYNC); cacheRCfg.setRebalanceBatchSize(1); cacheRCfg.setRebalanceBatchesPrefetchCount(Integer.MAX_VALUE); + cacheRCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + ((TcpCommunicationSpi)iCfg.getCommunicationSpi()).setSharedMemoryPort(-1);//Shmem fail fix for Integer.MAX_VALUE. CacheConfiguration cacheRCfg2 = new CacheConfiguration<>(DEFAULT_CACHE_NAME); @@ -150,6 +156,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest { cacheRCfg2.setCacheMode(CacheMode.REPLICATED); cacheRCfg2.setRebalanceMode(CacheRebalanceMode.SYNC); cacheRCfg2.setRebalanceOrder(4); + cacheRCfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); iCfg.setCacheConfiguration(cachePCfg, cachePCfg2, cacheRCfg, cacheRCfg2); @@ -226,6 +233,9 @@ protected void checkData(Ignite ignite, String name, int from, int iter) { * @throws Exception If failed. */ public void testSimpleRebalancing() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10560"); + IgniteKernal ignite = (IgniteKernal)startGrid(0); generateData(ignite, 0, 0); @@ -465,6 +475,9 @@ protected void awaitPartitionMessagesAbsent() throws Exception { * @throws Exception If failed. */ public void testComplexRebalancing() throws Exception { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10561"); + final Ignite ignite = startGrid(0); generateData(ignite, 0, 0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingUnmarshallingFailedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingUnmarshallingFailedSelfTest.java index 6d72a52b362de..de09e27f517a3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingUnmarshallingFailedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingUnmarshallingFailedSelfTest.java @@ -22,6 +22,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cache.query.annotations.QuerySqlField; @@ -109,6 +110,7 @@ public TestKey() { cfg.setCacheMode(CacheMode.PARTITIONED); cfg.setRebalanceMode(CacheRebalanceMode.SYNC); cfg.setBackups(0); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); iCfg.setCacheConfiguration(cfg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMetricsSelfTest.java index 16be6733a9392..374a765ba446e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/GridCacheReplicatedMetricsSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheTransactionalAbstractMetricsSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -29,6 +30,13 @@ public class GridCacheReplicatedMetricsSelfTest extends GridCacheTransactionalAb /** */ private static final int GRID_CNT = 2; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { CacheConfiguration cfg = super.cacheConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java index 84434696c9179..83771bad1409f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/DhtAndNearEvictionTest.java @@ -25,6 +25,7 @@ import javax.cache.integration.CacheWriterException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.eviction.lru.LruEvictionPolicyFactory; import org.apache.ignite.cache.store.CacheStoreAdapter; import org.apache.ignite.configuration.CacheConfiguration; @@ -35,6 +36,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridStringLogger; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** @@ -59,6 +61,13 @@ public class DhtAndNearEvictionTest extends GridCommonAbstractTest { return cfg; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.EVICTION); + + super.beforeTestsStarted(); + } + /** */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -85,6 +94,8 @@ public class DhtAndNearEvictionTest extends GridCommonAbstractTest { * */ public void testConcurrentWritesAndReadsWithReadThrough() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + startGrid(0); startGrid(1); @@ -97,6 +108,7 @@ public void testConcurrentWritesAndReadsWithReadThrough() throws Exception { ) .setReadThrough(true) .setCacheStoreFactory(DummyCacheStore.factoryOf()) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setBackups(1); grid(0).createCache(ccfg); @@ -142,6 +154,7 @@ public void testRebalancing() throws Exception { CacheConfiguration ccfg = new CacheConfiguration("mycache") .setOnheapCacheEnabled(true) .setEvictionPolicyFactory(new LruEvictionPolicyFactory<>(500)) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setNearConfiguration( new NearCacheConfiguration() .setNearEvictionPolicyFactory(new LruEvictionPolicyFactory<>(100)) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionConsistencySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionConsistencySelfTest.java index 5ff0be23a5cf5..6cbb04302d736 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionConsistencySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionConsistencySelfTest.java @@ -38,13 +38,14 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; -import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; /** * @@ -73,7 +74,7 @@ public class GridCacheConcurrentEvictionConsistencySelfTest extends GridCommonAb IgniteConfiguration c = super.getConfiguration(igniteInstanceName); c.getTransactionConfiguration().setDefaultTxConcurrency(PESSIMISTIC); - c.getTransactionConfiguration().setDefaultTxIsolation(READ_COMMITTED); + c.getTransactionConfiguration().setDefaultTxIsolation(REPEATABLE_READ); CacheConfiguration cc = defaultCacheConfiguration(); @@ -97,6 +98,13 @@ public class GridCacheConcurrentEvictionConsistencySelfTest extends GridCommonAb return c; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected long getTestTimeout() { return 5 * 60 * 1000; // 5 min. @@ -251,14 +259,21 @@ private void checkPolicyConsistency() throws Exception { int j = rnd.nextInt(keyCnt); - try (Transaction tx = ignite.transactions().txStart()) { - // Put or remove? - if (rnd.nextBoolean()) - cache.put(j, j); - else - cache.remove(j); - - tx.commit(); + while (true) { + try (Transaction tx = ignite.transactions().txStart()) { + // Put or remove? + if (rnd.nextBoolean()) + cache.put(j, j); + else + cache.remove(j); + + tx.commit(); + + break; + } + catch (Exception e) { + MvccFeatureChecker.assertMvccWriteConflict(e); + } } if (i != 0 && i % 5000 == 0) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionsSelfTest.java index 45d98bfc89e27..73763203531bb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheConcurrentEvictionsSelfTest.java @@ -22,6 +22,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.eviction.EvictionPolicy; import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy; import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy; @@ -32,13 +33,14 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; -import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; /** * @@ -64,7 +66,7 @@ public class GridCacheConcurrentEvictionsSelfTest extends GridCommonAbstractTest IgniteConfiguration c = super.getConfiguration(igniteInstanceName); c.getTransactionConfiguration().setDefaultTxConcurrency(PESSIMISTIC); - c.getTransactionConfiguration().setDefaultTxIsolation(READ_COMMITTED); + c.getTransactionConfiguration().setDefaultTxIsolation(REPEATABLE_READ); CacheConfiguration cc = defaultCacheConfiguration(); @@ -95,6 +97,13 @@ public class GridCacheConcurrentEvictionsSelfTest extends GridCommonAbstractTest plc = null; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } + /** * @throws Exception If failed. */ @@ -180,7 +189,7 @@ private void checkConcurrentPuts() throws Exception { if (i != 0 && i % 10000 == 0) // info("Puts count: " + i); - info("Stats [putsCnt=" + i + ", size=" + cache.size() + ']'); + info("Stats [putsCnt=" + i + ", size=" + cache.size(CachePeekMode.ONHEAP) + ']'); } return null; @@ -191,8 +200,10 @@ private void checkConcurrentPuts() throws Exception { fut.get(); - info("Test results [threadCnt=" + threadCnt + ", iterCnt=" + iterCnt + ", cacheSize=" + cache.size() + + info("Test results [threadCnt=" + threadCnt + ", iterCnt=" + iterCnt + ", cacheSize=" + cache.size(CachePeekMode.ONHEAP) + ", duration=" + (System.currentTimeMillis() - start) + ']'); + + assertTrue(cache.size(CachePeekMode.ONHEAP) <= 1000); } finally { stopAllGrids(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesAbstractSelfTest.java index f28e56b6fcda2..a04807a3fe848 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesAbstractSelfTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionConcurrency; @@ -172,6 +173,9 @@ private void checkPolicy0() throws Exception { for (TransactionIsolation isolation : TransactionIsolation.values()) { txIsolation = isolation; + if (MvccFeatureChecker.forcedMvcc() && !MvccFeatureChecker.isSupported(concurrency, isolation)) + continue; + Ignite g = startGrids(); IgniteCache cache = g.cache(DEFAULT_CACHE_NAME); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesLocalSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesLocalSelfTest.java index 018cc2e699dbc..c9cc00afdac43 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesLocalSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesLocalSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * @@ -38,4 +39,13 @@ public class GridCacheEmptyEntriesLocalSelfTest extends GridCacheEmptyEntriesAbs @Override public void testFifo() throws Exception { super.testFifo(); } + + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesPartitionedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesPartitionedSelfTest.java index 5d9faebd465dc..5ed4843901fcd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesPartitionedSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEmptyEntriesPartitionedSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.testframework.MvccFeatureChecker; /** * Test allow empty entries flag on partitioned cache. @@ -38,4 +39,11 @@ public class GridCacheEmptyEntriesPartitionedSelfTest extends GridCacheEmptyEntr @Override public void testFifo() throws Exception { super.testFifo(); } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionFilterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionFilterSelfTest.java index b432c2d584595..71a6295f39455 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionFilterSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionFilterSelfTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -105,6 +106,8 @@ public class GridCacheEvictionFilterSelfTest extends GridCommonAbstractTest { /** @throws Exception If failed. */ public void testLocal() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + mode = LOCAL; checkEvictionFilter(); @@ -119,6 +122,8 @@ public void testReplicated() throws Exception { /** @throws Exception If failed. */ public void testPartitioned() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.NEAR_CACHE); + mode = PARTITIONED; nearEnabled = true; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionLockUnlockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionLockUnlockSelfTest.java index 55b7b63691470..b5265a31e691c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionLockUnlockSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionLockUnlockSelfTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.concurrent.TimeUnit.SECONDS; @@ -95,6 +96,15 @@ public class GridCacheEvictionLockUnlockSelfTest extends GridCommonAbstractTest return c; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.ENTRY_LOCK); + + + super.beforeTestsStarted(); + } + /** @throws Exception If failed. */ public void testLocal() throws Exception { mode = LOCAL; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionTouchSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionTouchSelfTest.java index a91c5b6527e0d..384722e3b5708 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionTouchSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionTouchSelfTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -108,6 +109,13 @@ public class GridCacheEvictionTouchSelfTest extends GridCommonAbstractTest { return c; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.CACHE_STORE); + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { plc = null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearEvictionPolicySelfTest.java index 27295c69ca35e..8693d330c5aec 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearEvictionPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearEvictionPolicySelfTest.java @@ -32,6 +32,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; @@ -101,6 +102,18 @@ public void testTransactionalNearEvictionMaxSize() throws Exception { checkNearEvictionMaxSize(); } + /** + * @throws Exception If failed. + */ + public void testMvccTransactionalNearEvictionMaxSize() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + + checkNearEvictionMaxSize(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearOnlyNearEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearOnlyNearEvictionPolicySelfTest.java index a329e83f56252..3977c2e3a0ef9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearOnlyNearEvictionPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/lru/LruNearOnlyNearEvictionPolicySelfTest.java @@ -32,6 +32,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; @@ -111,6 +112,19 @@ public void testPartitionedTransactionalNearEvictionMaxSize() throws Exception { checkNearEvictionMaxSize(); } + /** + * @throws Exception If failed. + */ + public void testPartitionedMvccTransactionalNearEvictionMaxSize() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + cacheMode = PARTITIONED; + + checkNearEvictionMaxSize(); + } + /** * @throws Exception If failed. */ @@ -131,6 +145,19 @@ public void testReplicatedTransactionalNearEvictionMaxSize() throws Exception { checkNearEvictionMaxSize(); } + /** + * @throws Exception If failed. + */ + public void testReplicatedMvccTransactionalNearEvictionMaxSize() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7187"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + atomicityMode = TRANSACTIONAL_SNAPSHOT; + cacheMode = REPLICATED; + + checkNearEvictionMaxSize(); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java index b5aab69f5016e..ed70d8587887e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java @@ -34,10 +34,10 @@ public class PageEvictionDataStreamerTest extends PageEvictionMultinodeAbstractT } /** {@inheritDoc} */ - @Override protected void createCacheAndTestEvcition(CacheConfiguration cfg) throws Exception { - IgniteCache cache = clientGrid.getOrCreateCache(cfg); + @Override protected void createCacheAndTestEviction(CacheConfiguration cfg) throws Exception { + IgniteCache cache = clientGrid().getOrCreateCache(cfg); - try (IgniteDataStreamer ldr = clientGrid.dataStreamer(cfg.getName())) { + try (IgniteDataStreamer ldr = clientGrid().dataStreamer(cfg.getName())) { ldr.allowOverwrite(true); for (int i = 1; i <= ENTRIES; i++) { @@ -60,6 +60,6 @@ public class PageEvictionDataStreamerTest extends PageEvictionMultinodeAbstractT // Eviction started, no OutOfMemory occurred, success. assertTrue(resultingSize < ENTRIES); - clientGrid.destroyCache(cfg.getName()); + clientGrid().destroyCache(cfg.getName()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java index a451c36bf8d7f..857c31586a8fc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMetricTest.java @@ -44,6 +44,22 @@ public class PageEvictionMetricTest extends PageEvictionAbstractTest { * @throws Exception If failed. */ public void testPageEvictionMetric() throws Exception { + checkPageEvictionMetric(CacheAtomicityMode.ATOMIC); + } + + /** + * @throws Exception If failed. + */ + public void testPageEvictionMetricMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + + checkPageEvictionMetric(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } + + /** + * @throws Exception If failed. + */ + private void checkPageEvictionMetric(CacheAtomicityMode atomicityMode) throws Exception { IgniteEx ignite = startGrid(0); DataRegionMetricsImpl metrics = @@ -52,7 +68,7 @@ public void testPageEvictionMetric() throws Exception { metrics.enableMetrics(); CacheConfiguration cfg = cacheConfig("evict-metric", null, - CacheMode.PARTITIONED, CacheAtomicityMode.ATOMIC, CacheWriteSynchronizationMode.PRIMARY_SYNC); + CacheMode.PARTITIONED, atomicityMode, CacheWriteSynchronizationMode.PRIMARY_SYNC); IgniteCache cache = ignite.getOrCreateCache(cfg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java index e30f1cf040adf..d0d6f0b24f746 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeAbstractTest.java @@ -25,7 +25,6 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.testframework.MvccFeatureChecker; /** * @@ -42,19 +41,19 @@ public abstract class PageEvictionMultinodeAbstractTest extends PageEvictionAbst private static final CacheWriteSynchronizationMode[] WRITE_MODES = {CacheWriteSynchronizationMode.PRIMARY_SYNC, CacheWriteSynchronizationMode.FULL_SYNC, CacheWriteSynchronizationMode.FULL_ASYNC}; - /** Client grid. */ - Ignite clientGrid; - /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10448"); - startGridsMultiThreaded(4, false); - clientGrid = startGrid("client"); + startGrid("client"); } + /** + * @return Client grid. + */ + Ignite clientGrid() { + return grid("client"); + } /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -82,19 +81,34 @@ public void testPageEviction() throws Exception { CacheConfiguration cfg = cacheConfig( "evict" + i + j + k, null, CACHE_MODES[i], ATOMICITY_MODES[j], WRITE_MODES[k]); - createCacheAndTestEvcition(cfg); + createCacheAndTestEviction(cfg); } } } } } + /** + * @throws Exception If failed. + */ + public void testPageEvictionMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + + for (int i = 0; i < CACHE_MODES.length; i++) { + CacheConfiguration cfg = cacheConfig( + "evict" + i, null, CACHE_MODES[i], CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, + CacheWriteSynchronizationMode.FULL_SYNC); + + createCacheAndTestEviction(cfg); + } + } + /** * @param cfg Config. * @throws Exception If failed. */ - protected void createCacheAndTestEvcition(CacheConfiguration cfg) throws Exception { - IgniteCache cache = clientGrid.getOrCreateCache(cfg); + protected void createCacheAndTestEviction(CacheConfiguration cfg) throws Exception { + IgniteCache cache = clientGrid().getOrCreateCache(cfg); for (int i = 1; i <= ENTRIES; i++) { ThreadLocalRandom r = ThreadLocalRandom.current(); @@ -122,6 +136,6 @@ else if (r.nextInt() % 13 == 0) // Eviction started, no OutOfMemory occurred, success. assertTrue(resultingSize < ENTRIES * 10 / 11); - clientGrid.destroyCache(cfg.getName()); + clientGrid().destroyCache(cfg.getName()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java index 9a96a642a07f3..bb36d2a84e52d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java @@ -48,7 +48,7 @@ public class PageEvictionMultinodeMixedRegionsTest extends PageEvictionMultinode super.beforeTestsStarted(); - clientGrid.active(true); + clientGrid().cluster().active(true); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPagesRecyclingAndReusingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPagesRecyclingAndReusingTest.java index 9c777fbe36c80..88db8fdd9fbf7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPagesRecyclingAndReusingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionPagesRecyclingAndReusingTest.java @@ -82,6 +82,35 @@ public void testPagesRecyclingAndReusingTxLocal() throws Exception { testPagesRecyclingAndReusing(CacheAtomicityMode.TRANSACTIONAL, CacheMode.LOCAL); } + /** + * @throws Exception If failed. + */ + public void testPagesRecyclingAndReusingMvccTxPartitioned() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + + testPagesRecyclingAndReusing(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.PARTITIONED); + } + + + /** + * @throws Exception If failed. + */ + public void testPagesRecyclingAndReusingMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + + testPagesRecyclingAndReusing(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.REPLICATED); + } + + /** + * @throws Exception If failed. + */ + public void testPagesRecyclingAndReusingMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + testPagesRecyclingAndReusing(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.LOCAL); + } + /** * @param atomicityMode Atomicity mode. * @param cacheMode Cache mode. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughTest.java index ff713616c742f..e17d2cccc83a8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionReadThroughTest.java @@ -67,6 +67,37 @@ public void testEvictionWithReadThroughTxLocal() throws Exception { testEvictionWithReadThrough(CacheAtomicityMode.TRANSACTIONAL, CacheMode.LOCAL); } + /** + * @throws Exception If failed. + */ + public void testEvictionWithReadThroughMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + testEvictionWithReadThrough(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.REPLICATED); + } + + /** + * @throws Exception If failed. + */ + public void testEvictionWithReadThroughMvccTxPartitioned() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + testEvictionWithReadThrough(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.PARTITIONED); + } + + /** + * @throws Exception If failed. + */ + public void testEvictionWithReadThroughMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + fail("https://issues.apache.org/jira/browse/IGNITE-8582"); + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + testEvictionWithReadThrough(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.LOCAL); + } + /** * @param atomicityMode Atomicity mode. * @param cacheMode Cache mode. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionTouchOrderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionTouchOrderTest.java index 43356490fe21e..68c39dd59182e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionTouchOrderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionTouchOrderTest.java @@ -70,6 +70,36 @@ public void testTouchOrderWithFairFifoEvictionTxLocal() throws Exception { testTouchOrderWithFairFifoEviction(CacheAtomicityMode.TRANSACTIONAL, CacheMode.LOCAL); } + /** + * @throws Exception If failed. + */ + public void testTouchOrderWithFairFifoEvictionMvccTxReplicated() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + testTouchOrderWithFairFifoEviction(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.REPLICATED); + } + + /** + * @throws Exception If failed. + */ + public void testTouchOrderWithFairFifoEvictionMvccTxPartitioned() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + + testTouchOrderWithFairFifoEviction(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.PARTITIONED); + } + + /** + * @throws Exception If failed. + */ + public void testTouchOrderWithFairFifoEvictionMvccTxLocal() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-7956"); + fail("https://issues.apache.org/jira/browse/IGNITE-9530"); + + testTouchOrderWithFairFifoEviction(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT, CacheMode.LOCAL); + } + /** * @param atomicityMode Atomicity mode. * @param cacheMode Cache mode. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithRebalanceAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithRebalanceAbstractTest.java index 3ad104b031160..39e12cbaa6388 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithRebalanceAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionWithRebalanceAbstractTest.java @@ -32,10 +32,26 @@ public abstract class PageEvictionWithRebalanceAbstractTest extends PageEviction * @throws Exception If failed. */ public void testEvictionWithRebalance() throws Exception { + checkEvictionWithRebalance(CacheAtomicityMode.ATOMIC); + } + + /** + * @throws Exception If failed. + */ + public void testEvictionWithRebalanceMvcc() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-10448"); + + checkEvictionWithRebalance(CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT); + } + + /** + * @throws Exception If failed. + */ + public void checkEvictionWithRebalance(CacheAtomicityMode atomicityMode) throws Exception { startGridsMultiThreaded(4); CacheConfiguration cfg = cacheConfig("evict-rebalance", null, CacheMode.PARTITIONED, - CacheAtomicityMode.ATOMIC, CacheWriteSynchronizationMode.PRIMARY_SYNC); + atomicityMode, CacheWriteSynchronizationMode.PRIMARY_SYNC); IgniteCache cache = ignite(0).getOrCreateCache(cfg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIteratorsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIteratorsSelfTest.java index 6af80ca2a1869..6fa8edc00327b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIteratorsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalIteratorsSelfTest.java @@ -19,6 +19,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.internal.processors.cache.GridCacheAbstractIteratorsSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -40,4 +41,11 @@ public class GridCacheLocalIteratorsSelfTest extends GridCacheAbstractIteratorsS @Override protected int entryCount() { return 1000; } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + + super.beforeTestsStarted(); + } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMetricsSelfTest.java index 16a2e3393fd75..d66799e38f4a3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalMetricsSelfTest.java @@ -20,6 +20,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.GridCacheTransactionalAbstractMetricsSelfTest; +import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -30,6 +31,15 @@ public class GridCacheLocalMetricsSelfTest extends GridCacheTransactionalAbstrac /** */ private static final int GRID_CNT = 1; + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.LOCAL_CACHE); + MvccFeatureChecker.failIfNotSupported(MvccFeatureChecker.Feature.METRICS); + + + super.beforeTestsStarted(); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration c = super.getConfiguration(igniteInstanceName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheAssignmentNodeRestartsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheAssignmentNodeRestartsTest.java index 032b422a60bc8..a68bc7ba8c4dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheAssignmentNodeRestartsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheAssignmentNodeRestartsTest.java @@ -45,6 +45,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; @@ -129,6 +130,9 @@ private CacheConfiguration cacheConfiguration(String name, */ public void testAssignmentAfterRestarts() throws Exception { try { + if (MvccFeatureChecker.forcedMvcc()) + fail("https://issues.apache.org/jira/browse/IGNITE-10582"); + System.setProperty(IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC, "true"); final int gridsCnt = 5; diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index ad9658d3593ca..aa169575eeb74 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -17,10 +17,12 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.GridCachePreloadingEvictionsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearEvictionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearEvictionSelfTest; +import org.apache.ignite.internal.processors.cache.eviction.DhtAndNearEvictionTest; import org.apache.ignite.internal.processors.cache.eviction.GridCacheConcurrentEvictionConsistencySelfTest; import org.apache.ignite.internal.processors.cache.eviction.GridCacheConcurrentEvictionsSelfTest; import org.apache.ignite.internal.processors.cache.eviction.GridCacheEmptyEntriesLocalSelfTest; @@ -29,7 +31,6 @@ import org.apache.ignite.internal.processors.cache.eviction.GridCacheEvictionFilterSelfTest; import org.apache.ignite.internal.processors.cache.eviction.GridCacheEvictionLockUnlockSelfTest; import org.apache.ignite.internal.processors.cache.eviction.GridCacheEvictionTouchSelfTest; -import org.apache.ignite.internal.processors.cache.eviction.DhtAndNearEvictionTest; import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPolicyFactorySelfTest; import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicyFactorySelfTest; @@ -49,53 +50,54 @@ import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithRebalanceTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite for cache eviction. */ public class IgniteCacheEvictionSelfTestSuite extends TestSuite { /** + * @param ignoredTests Ignored tests. * @return Cache eviction test suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Ignite Cache Eviction Test Suite"); - suite.addTest(new TestSuite(FifoEvictionPolicySelfTest.class)); - suite.addTest(new TestSuite(SortedEvictionPolicySelfTest.class)); - suite.addTest(new TestSuite(LruEvictionPolicySelfTest.class)); - suite.addTest(new TestSuite(FifoEvictionPolicyFactorySelfTest.class)); - suite.addTest(new TestSuite(SortedEvictionPolicyFactorySelfTest.class)); - suite.addTest(new TestSuite(LruEvictionPolicyFactorySelfTest.class)); - suite.addTest(new TestSuite(LruNearEvictionPolicySelfTest.class)); - suite.addTest(new TestSuite(LruNearOnlyNearEvictionPolicySelfTest.class)); - suite.addTest(new TestSuite(GridCacheNearEvictionSelfTest.class)); - suite.addTest(new TestSuite(GridCacheAtomicNearEvictionSelfTest.class)); - suite.addTest(new TestSuite(GridCacheEvictionFilterSelfTest.class)); - suite.addTest(new TestSuite(GridCacheConcurrentEvictionsSelfTest.class)); - suite.addTest(new TestSuite(GridCacheConcurrentEvictionConsistencySelfTest.class)); - suite.addTest(new TestSuite(GridCacheEvictionTouchSelfTest.class)); - suite.addTest(new TestSuite(GridCacheEvictionLockUnlockSelfTest.class)); - suite.addTest(new TestSuite(GridCachePreloadingEvictionsSelfTest.class)); - suite.addTest(new TestSuite(GridCacheEmptyEntriesPartitionedSelfTest.class)); - suite.addTest(new TestSuite(GridCacheEmptyEntriesLocalSelfTest.class)); - suite.addTest(new TestSuite(GridCacheEvictableEntryEqualsSelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, FifoEvictionPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, SortedEvictionPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, LruEvictionPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, FifoEvictionPolicyFactorySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, SortedEvictionPolicyFactorySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, LruEvictionPolicyFactorySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, LruNearEvictionPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, LruNearOnlyNearEvictionPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearEvictionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicNearEvictionSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEvictionFilterSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheConcurrentEvictionsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheConcurrentEvictionConsistencySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEvictionTouchSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEvictionLockUnlockSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePreloadingEvictionsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEmptyEntriesPartitionedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEmptyEntriesLocalSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheEvictableEntryEqualsSelfTest.class, ignoredTests); - suite.addTest(new TestSuite(RandomLruPageEvictionMultinodeTest.class)); - suite.addTest(new TestSuite(RandomLruNearEnabledPageEvictionMultinodeTest.class)); - suite.addTest(new TestSuite(Random2LruPageEvictionMultinodeTest.class)); - suite.addTest(new TestSuite(Random2LruNearEnabledPageEvictionMultinodeTest.class)); - suite.addTest(new TestSuite(RandomLruPageEvictionWithRebalanceTest.class)); - suite.addTest(new TestSuite(Random2LruPageEvictionWithRebalanceTest.class)); - suite.addTest(new TestSuite(PageEvictionTouchOrderTest.class)); - suite.addTest(new TestSuite(PageEvictionReadThroughTest.class)); - suite.addTest(new TestSuite(PageEvictionDataStreamerTest.class)); + GridTestUtils.addTestIfNeeded(suite, RandomLruPageEvictionMultinodeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, RandomLruNearEnabledPageEvictionMultinodeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Random2LruPageEvictionMultinodeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Random2LruNearEnabledPageEvictionMultinodeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, RandomLruPageEvictionWithRebalanceTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, Random2LruPageEvictionWithRebalanceTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionTouchOrderTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionReadThroughTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PageEvictionDataStreamerTest.class, ignoredTests); - suite.addTest(new TestSuite(PageEvictionMetricTest.class)); + GridTestUtils.addTestIfNeeded(suite, PageEvictionMetricTest.class, ignoredTests); - suite.addTest(new TestSuite(PageEvictionPagesRecyclingAndReusingTest.class)); + GridTestUtils.addTestIfNeeded(suite, PageEvictionPagesRecyclingAndReusingTest.class, ignoredTests); - suite.addTest(new TestSuite(DhtAndNearEvictionTest.class)); + GridTestUtils.addTestIfNeeded(suite, DhtAndNearEvictionTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheIteratorsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheIteratorsSelfTestSuite.java index 6fb3b486cb1a5..3453955aa0011 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheIteratorsSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheIteratorsSelfTestSuite.java @@ -17,25 +17,27 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedIteratorsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedIteratorsSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalIteratorsSelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Cache iterators test suite. */ public class IgniteCacheIteratorsSelfTestSuite extends TestSuite { /** + * @param ignoredTests Ignored tests. * @return Cache iterators test suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Cache Iterators Test Suite"); - suite.addTest(new TestSuite(GridCacheLocalIteratorsSelfTest.class)); - suite.addTest(new TestSuite(GridCacheReplicatedIteratorsSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedIteratorsSelfTest.class)); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalIteratorsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheReplicatedIteratorsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedIteratorsSelfTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java index b6dcb21af8595..4c8464e93b172 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.TransactionMetricsMxBeanImplTest; import org.apache.ignite.internal.processors.cache.CacheGroupsMetricsRebalanceTest; @@ -38,45 +39,46 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalTckMetricsSelfTestImpl; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicMetricsNoReadThroughSelfTest; import org.apache.ignite.internal.processors.cache.local.GridCacheLocalMetricsSelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite for cache metrics. */ public class IgniteCacheMetricsSelfTestSuite extends TestSuite { /** + * @param ignoredTests Ignored tests. * @return Cache metrics test suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Cache Metrics Test Suite"); - suite.addTestSuite(GridCacheLocalMetricsSelfTest.class); - suite.addTestSuite(GridCacheLocalAtomicMetricsNoReadThroughSelfTest.class); - suite.addTestSuite(GridCacheNearMetricsSelfTest.class); - suite.addTestSuite(GridCacheNearAtomicMetricsSelfTest.class); - suite.addTestSuite(GridCacheReplicatedMetricsSelfTest.class); - suite.addTestSuite(GridCachePartitionedMetricsSelfTest.class); - suite.addTestSuite(GridCachePartitionedHitsAndMissesSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheLocalAtomicMetricsNoReadThroughSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheNearAtomicMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheReplicatedMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCachePartitionedHitsAndMissesSelfTest.class, ignoredTests); // Atomic cache. - suite.addTestSuite(GridCacheAtomicLocalMetricsSelfTest.class); - suite.addTestSuite(GridCacheAtomicLocalMetricsNoStoreSelfTest.class); - suite.addTestSuite(GridCacheAtomicReplicatedMetricsSelfTest.class); - suite.addTestSuite(GridCacheAtomicPartitionedMetricsSelfTest.class); - suite.addTestSuite(GridCacheAtomicPartitionedTckMetricsSelfTestImpl.class); - suite.addTestSuite(GridCacheAtomicLocalTckMetricsSelfTestImpl.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicLocalMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicLocalMetricsNoStoreSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicReplicatedMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicPartitionedMetricsSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicPartitionedTckMetricsSelfTestImpl.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheAtomicLocalTckMetricsSelfTestImpl.class, ignoredTests); - suite.addTestSuite(CacheGroupsMetricsRebalanceTest.class); - suite.addTestSuite(CacheValidatorMetricsTest.class); - suite.addTestSuite(CacheMetricsEntitiesCountTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheGroupsMetricsRebalanceTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheValidatorMetricsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheMetricsEntitiesCountTest.class, ignoredTests); // Cluster wide metrics. - suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class); - suite.addTestSuite(OffheapCacheMetricsForClusterGroupSelfTest.class); + GridTestUtils.addTestIfNeeded(suite, CacheMetricsForClusterGroupSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, OffheapCacheMetricsForClusterGroupSelfTest.class, ignoredTests); - suite.addTestSuite(TransactionMetricsMxBeanImplTest.class); + GridTestUtils.addTestIfNeeded(suite, TransactionMetricsMxBeanImplTest.class, ignoredTests); - suite.addTestSuite(GridEvictionPolicyMBeansTest.class); + GridTestUtils.addTestIfNeeded(suite, GridEvictionPolicyMBeansTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java index 403b5a026f05a..a41e565f6f2ce 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite7.java @@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersMvccTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest; import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest; /** @@ -45,6 +46,9 @@ public static TestSuite suite() { HashSet ignoredTests = new HashSet<>(128); + // Skip classes that already contains Mvcc tests + ignoredTests.add(PageEvictionMultinodeMixedRegionsTest.class); + // Other non-tx tests. ignoredTests.add(CheckpointBufferDeadlockTest.class);// ignoredTests.add(AuthenticationConfigurationClusterTest.class);// diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite8.java new file mode 100644 index 0000000000000..03ce6c18612d4 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite8.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.testsuites; + +import java.util.HashSet; +import junit.framework.TestSuite; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorNearPartitionedAtomicCacheGroupsTest; +import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorNearPartitionedAtomicCacheTest; +import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorPartitionedAtomicCacheGroupsTest; +import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorPartitionedAtomicCacheTest; +import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedAtomicCacheGroupsTest; +import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedAtomicCacheTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearEvictionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedMetricsSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedTckMetricsSelfTestImpl; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheMvccNearEvictionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearAtomicMetricsSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearEvictionSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheAtomicReplicatedMetricsSelfTest; +import org.apache.ignite.internal.processors.cache.eviction.GridCacheEvictableEntryEqualsSelfTest; +import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPolicyFactorySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicyFactorySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionDataStreamerTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMetricTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionPagesRecyclingAndReusingTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionTouchOrderTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruNearEnabledPageEvictionMultinodeTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionMultinodeTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruPageEvictionWithRebalanceTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruNearEnabledPageEvictionMultinodeTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionMultinodeTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.RandomLruPageEvictionWithRebalanceTest; +import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyFactorySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalMetricsNoStoreSelfTest; +import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalMetricsSelfTest; +import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalTckMetricsSelfTestImpl; +import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicMetricsNoReadThroughSelfTest; + +/** + * + */ +public class IgniteCacheMvccTestSuite8 extends TestSuite { + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite() { + System.setProperty(IgniteSystemProperties.IGNITE_FORCE_MVCC_MODE_IN_TESTS, "true"); + + HashSet ignoredTests = new HashSet<>(128); + + // Skip classes that already contains Mvcc tests + ignoredTests.add(LruNearEvictionPolicySelfTest.class); + ignoredTests.add(LruNearOnlyNearEvictionPolicySelfTest.class); + ignoredTests.add(RandomLruPageEvictionMultinodeTest.class); + ignoredTests.add(RandomLruNearEnabledPageEvictionMultinodeTest.class); + ignoredTests.add(PageEvictionDataStreamerTest.class); + ignoredTests.add(Random2LruPageEvictionMultinodeTest.class); + ignoredTests.add(Random2LruNearEnabledPageEvictionMultinodeTest.class); + ignoredTests.add(RandomLruPageEvictionWithRebalanceTest.class); + ignoredTests.add(Random2LruPageEvictionWithRebalanceTest.class); + ignoredTests.add(PageEvictionTouchOrderTest.class); + ignoredTests.add(PageEvictionReadThroughTest.class); + ignoredTests.add(PageEvictionMetricTest.class); + ignoredTests.add(PageEvictionPagesRecyclingAndReusingTest.class); + + // Irrelevant Tx tests. + ignoredTests.add(GridCacheEvictableEntryEqualsSelfTest.class); + + // Atomic cache tests. + ignoredTests.add(GridCacheLocalAtomicMetricsNoReadThroughSelfTest.class); + ignoredTests.add(GridCacheNearAtomicMetricsSelfTest.class); + ignoredTests.add(GridCacheAtomicLocalMetricsSelfTest.class); + ignoredTests.add(GridCacheAtomicLocalMetricsNoStoreSelfTest.class); + ignoredTests.add(GridCacheAtomicReplicatedMetricsSelfTest.class); + ignoredTests.add(GridCacheAtomicPartitionedMetricsSelfTest.class); + ignoredTests.add(GridCacheAtomicPartitionedTckMetricsSelfTestImpl.class); + ignoredTests.add(GridCacheAtomicLocalTckMetricsSelfTestImpl.class); + ignoredTests.add(IgniteTopologyValidatorPartitionedAtomicCacheTest.class); + ignoredTests.add(IgniteTopologyValidatorNearPartitionedAtomicCacheTest.class); + ignoredTests.add(IgniteTopologyValidatorReplicatedAtomicCacheTest.class); + ignoredTests.add(IgniteTopologyValidatorNearPartitionedAtomicCacheGroupsTest.class); + ignoredTests.add(IgniteTopologyValidatorPartitionedAtomicCacheGroupsTest.class); + ignoredTests.add(IgniteTopologyValidatorReplicatedAtomicCacheGroupsTest.class); + + // Other non-tx tests. + ignoredTests.add(FifoEvictionPolicySelfTest.class); + ignoredTests.add(SortedEvictionPolicySelfTest.class); + ignoredTests.add(LruEvictionPolicySelfTest.class); + ignoredTests.add(FifoEvictionPolicyFactorySelfTest.class); + ignoredTests.add(SortedEvictionPolicyFactorySelfTest.class); + ignoredTests.add(LruEvictionPolicyFactorySelfTest.class); + ignoredTests.add(GridCacheAtomicNearEvictionSelfTest.class); + ignoredTests.add(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class); + + // Skip classes which Mvcc implementations are added in this method below. + // TODO IGNITE-10175: refactor these tests (use assume) to support both mvcc and non-mvcc modes after moving to JUnit4/5. + ignoredTests.add(GridCacheNearEvictionSelfTest.class); // See GridCacheMvccNearEvictionSelfTest + + TestSuite suite = new TestSuite("IgniteCache Mvcc Test Suite part 8"); + + suite.addTest(IgniteCacheTestSuite8.suite(ignoredTests)); + + // Add Mvcc clones. + suite.addTestSuite(GridCacheMvccNearEvictionSelfTest.class); + + return suite; + } + +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java index 8ba955efe3a49..fa4cf244e0caf 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite8.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.GridCacheOrderedPreloadingSelfTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest; @@ -25,6 +26,7 @@ import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Test suite. @@ -32,31 +34,37 @@ public class IgniteCacheTestSuite8 extends TestSuite { /** * @return IgniteCache test suite. - * @throws Exception Thrown in case of the failure. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite() { + return suite(null); + } + + /** + * @return IgniteCache test suite. + */ + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("IgniteCache Test Suite part 8"); // Cache metrics. - suite.addTest(IgniteCacheMetricsSelfTestSuite.suite()); + suite.addTest(IgniteCacheMetricsSelfTestSuite.suite(ignoredTests)); // Topology validator. - suite.addTest(IgniteTopologyValidatorTestSuite.suite()); + suite.addTest(IgniteTopologyValidatorTestSuite.suite(ignoredTests)); // Eviction. - suite.addTest(IgniteCacheEvictionSelfTestSuite.suite()); + suite.addTest(IgniteCacheEvictionSelfTestSuite.suite(ignoredTests)); // Iterators. - suite.addTest(IgniteCacheIteratorsSelfTestSuite.suite()); + suite.addTest(IgniteCacheIteratorsSelfTestSuite.suite(ignoredTests)); // Rebalancing. - suite.addTestSuite(GridCacheOrderedPreloadingSelfTest.class); - suite.addTestSuite(GridCacheRebalancingSyncSelfTest.class); - suite.addTestSuite(GridCacheRebalancingSyncCheckDataTest.class); - suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class); - suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class); - suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class); - suite.addTestSuite(GridCacheRebalancingCancelTest.class); + GridTestUtils.addTestIfNeeded(suite, GridCacheOrderedPreloadingSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingSyncSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingSyncCheckDataTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingUnmarshallingFailedSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingAsyncSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingCancelTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java index 1c9b8525c8be0..4c3053eba8d21 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.testsuites; +import java.util.Collection; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorGridSplitCacheTest; import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorNearPartitionedAtomicCacheGroupsTest; @@ -31,33 +32,34 @@ import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedAtomicCacheTest; import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedTxCacheGroupsTest; import org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedTxCacheTest; +import org.apache.ignite.testframework.GridTestUtils; /** * Topology validator test suite. */ public class IgniteTopologyValidatorTestSuite extends TestSuite { /** + * @param ignoredTests Ignored tests. * @return Topology validator tests suite. - * @throws Exception If failed. */ - public static TestSuite suite() throws Exception { + public static TestSuite suite(Collection ignoredTests) { TestSuite suite = new TestSuite("Topology validator Test Suite"); - suite.addTest(new TestSuite(IgniteTopologyValidatorNearPartitionedAtomicCacheTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorNearPartitionedTxCacheTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorPartitionedAtomicCacheTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorPartitionedTxCacheTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorReplicatedAtomicCacheTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorReplicatedTxCacheTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorNearPartitionedAtomicCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorNearPartitionedTxCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorPartitionedAtomicCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorPartitionedTxCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorReplicatedAtomicCacheTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorReplicatedTxCacheTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteTopologyValidatorNearPartitionedAtomicCacheGroupsTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorPartitionedAtomicCacheGroupsTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorPartitionedTxCacheGroupsTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorReplicatedAtomicCacheGroupsTest.class)); - suite.addTest(new TestSuite(IgniteTopologyValidatorReplicatedTxCacheGroupsTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorNearPartitionedAtomicCacheGroupsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorNearPartitionedTxCacheGroupsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorPartitionedAtomicCacheGroupsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorPartitionedTxCacheGroupsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorReplicatedAtomicCacheGroupsTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorReplicatedTxCacheGroupsTest.class, ignoredTests); - suite.addTest(new TestSuite(IgniteTopologyValidatorGridSplitCacheTest.class)); + GridTestUtils.addTestIfNeeded(suite, IgniteTopologyValidatorGridSplitCacheTest.class, ignoredTests); return suite; } From 3a8ebd38f5e0110c9baabf6d07179542ea5c5fa4 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Wed, 19 Dec 2018 15:50:31 +0300 Subject: [PATCH 279/403] IGNITE-10262: MVCC TX: Client operation may hangs if all data nodes left the grid. This closes #5693. --- .../near/GridNearTxEnlistFuture.java | 3 +- .../near/GridNearTxQueryEnlistFuture.java | 5 + .../GridNearTxQueryResultsEnlistFuture.java | 3 +- .../IgniteClientCacheStartFailoverTest.java | 2 - .../processors/query/h2/IgniteH2Indexing.java | 12 +- ...vccAbstractSqlCoordinatorFailoverTest.java | 161 ++++++++++++++++++ 6 files changed, 180 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java index 3b72294fe2d5e..1044df5de51ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java @@ -31,6 +31,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.cache.CacheEntryPredicate; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMessage; @@ -218,7 +219,7 @@ private Collection continueLoop(@Nullable UUID nodeId) throws IgniteCheck ClusterNode node = cctx.affinity().primaryByKey(key, topVer); if (node == null) - throw new ClusterTopologyCheckedException("Failed to get primary node " + + throw new ClusterTopologyServerNotFoundException("Failed to get primary node " + "[topVer=" + topVer + ", key=" + key + ']'); tx.markQueryEnlisted(null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java index 70fbd1fce793b..faf84ef96d44f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java @@ -23,6 +23,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.affinity.AffinityAssignment; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMessage; @@ -120,6 +121,10 @@ protected GridNearTxQueryEnlistFuture( updateMappings(pNode); } + if (primary.isEmpty()) + throw new ClusterTopologyServerNotFoundException("Failed to find data nodes for cache (all partition " + + "nodes left the grid)."); + boolean locallyMapped = primary.contains(cctx.localNode()); if (locallyMapped) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java index 11ae966b9160e..07987ddea90ef 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java @@ -31,6 +31,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMessage; import org.apache.ignite.internal.processors.cache.KeyCacheObject; @@ -204,7 +205,7 @@ private Collection continueLoop(@Nullable UUID nodeId) throws IgniteCheck ClusterNode node = cctx.affinity().primaryByPartition(key.partition(), topVer); if (node == null) - throw new ClusterTopologyCheckedException("Failed to get primary node " + + throw new ClusterTopologyServerNotFoundException("Failed to get primary node " + "[topVer=" + topVer + ", key=" + key + ']'); if (!sequential) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java index c8025528764dc..6cb3a90fccf22 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java @@ -182,8 +182,6 @@ public void testClientStartLastServerFailsTx() throws Exception { * @throws Exception If failed. */ public void testClientStartLastServerFailsMvccTx() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10262"); - clientStartLastServerFails(TRANSACTIONAL_SNAPSHOT); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index ae192c1d355d9..86a122611530f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1804,8 +1804,16 @@ private Iterable> runQueryTwoStep( return new Iterable>() { @SuppressWarnings("NullableProblems") @Override public Iterator> iterator() { - return rdcQryExec.query(schemaName, qry, keepCacheObj, enforceJoinOrder, opTimeout, - cancel, params, parts, lazy, tracker); + try { + return rdcQryExec.query(schemaName, qry, keepCacheObj, enforceJoinOrder, opTimeout, + cancel, params, parts, lazy, tracker); + } + catch (Throwable e) { + if (tracker != null) + tracker.onDone(); + + throw e; + } } }; } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java index c449ee20e390b..9ecb1549a97f6 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java @@ -17,6 +17,23 @@ package org.apache.ignite.internal.processors.cache.mvcc; +import java.util.concurrent.Callable; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheServerNotFoundException; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteNodeAttributes; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.GridTestUtils; + +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.DML; @@ -135,4 +152,148 @@ public void testCoordinatorChangeActiveQueryClientFails_Simple() throws Exceptio public void testCoordinatorChangeActiveQueryClientFails_SimpleScan() throws Exception { checkCoordinatorChangeActiveQueryClientFails_Simple(new InitIndexing(Integer.class, Integer.class), SCAN, DML); } + + /** + * @throws Exception If failed. + */ + public void testStartLastServerFails() throws Exception { + testSpi = true; + + startGrids(3); + + CacheConfiguration cfg = cacheConfiguration(cacheMode(), FULL_SYNC, 0, DFLT_PARTITION_COUNT) + .setIndexedTypes(Integer.class, Integer.class); + + cfg.setNodeFilter(new TestNodeFilter(getTestIgniteInstanceName(1))); + + Ignite srv1 = ignite(1); + + srv1.createCache(cfg); + + client = true; + + final Ignite c = startGrid(3); + + client = false; + + TestRecordingCommunicationSpi.spi(srv1).blockMessages(GridDhtAffinityAssignmentResponse.class, c.name()); + + IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Void call() throws Exception { + c.cache(DEFAULT_CACHE_NAME); + + return null; + } + }, "start-cache"); + + U.sleep(1000); + + assertFalse(fut.isDone()); + + stopGrid(1); + + fut.get(); + + final IgniteCache clientCache = c.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 10; i++) { + final int k = i; + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.get(k); + + return null; + } + }, CacheServerNotFoundException.class, null); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.put(k, k); + + return null; + } + }, CacheServerNotFoundException.class, null); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.remove(k); + + return null; + } + }, CacheServerNotFoundException.class, null); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("SELECT * FROM INTEGER")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("SELECT * FROM INTEGER ORDER BY _val")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("DELETE FROM Integer WHERE 1 = 1")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("INSERT INTO Integer (_key, _val) VALUES (1, 2)")).getAll(); + + return null; + } + }, CacheException.class, "Failed to get primary node"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("UPDATE Integer SET _val=42 WHERE _key IN (SELECT DISTINCT _val FROM INTEGER)")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + } + + startGrid(1); + + awaitPartitionMapExchange(); + + for (int i = 0; i < 100; i++) { + assertNull(clientCache.get(i)); + + clientCache.put(i, i); + + assertEquals(i, clientCache.get(i)); + } + } + + /** + * + */ + private static class TestNodeFilter implements IgnitePredicate { + /** */ + private final String includeName; + + /** + * @param includeName Node to include. + */ + public TestNodeFilter(String includeName) { + this.includeName = includeName; + } + + /** {@inheritDoc} */ + @Override public boolean apply(ClusterNode node) { + return includeName.equals(node.attribute(IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME)); + } + } } From a7eff7a5baa85a73a3e2afb3ddf6b188422cad48 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Wed, 19 Dec 2018 15:54:59 +0300 Subject: [PATCH 280/403] IGNITE-10264: MVCC TX: Unmute tests. This closes #5685. --- .../processors/cache/IgniteCacheIncrementTxTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java index 918d655e013ac..b3a505597b8dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheIncrementTxTest.java @@ -36,7 +36,6 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -89,9 +88,6 @@ public void testIncrementTxTopologyChange0() throws Exception { * @throws Exception If failed. */ public void testIncrementTxTopologyChange1() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10264"); - nodeJoin(cacheConfiguration(1)); } @@ -99,9 +95,6 @@ public void testIncrementTxTopologyChange1() throws Exception { * @throws Exception If failed. */ public void testIncrementTxTopologyChange2() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10264"); - nodeJoin(cacheConfiguration(2)); } From 25dc16116da695f6f77dfaf2a40c50576bdcc907 Mon Sep 17 00:00:00 2001 From: rkondakov Date: Mon, 24 Dec 2018 18:42:18 +0300 Subject: [PATCH 281/403] IGNITE-10448: MVCC: Fixed NPE on data page eviction. This closes #5704. --- .../processors/cache/GridCacheProcessor.java | 14 +++++++ .../CacheMvccConfigurationValidationTest.java | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 91f3f17dfe48e..5df8f3e783b1d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -55,6 +55,7 @@ import org.apache.ignite.cache.store.CacheStoreSessionListener; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.configuration.FileSystemConfiguration; @@ -544,6 +545,19 @@ private void validate(IgniteConfiguration c, assertParameter(cc.getInterceptor() == null, "interceptor cannot be used with TRANSACTIONAL_SNAPSHOT atomicity mode"); + + // Disable in-memory evictions for mvcc cache. TODO IGNITE-10738 + String memPlcName = cc.getDataRegionName(); + DataRegion dataRegion = sharedCtx.database().dataRegion(memPlcName); + + if (dataRegion != null && !dataRegion.config().isPersistenceEnabled() && + dataRegion.config().getPageEvictionMode() != DataPageEvictionMode.DISABLED) { + throw new IgniteCheckedException("Data pages evictions cannot be used with TRANSACTIONAL_SNAPSHOT " + + "cache atomicity mode for in-memory regions. Please, either disable evictions or enable " + + "persistence for data regions with TRANSACTIONAL_SNAPSHOT caches. [cacheName=" + cc.getName() + + ", dataRegionName=" + memPlcName + ", pageEvictionMode=" + + dataRegion.config().getPageEvictionMode() + ']'); + } } if (cc.isWriteBehindEnabled()) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java index f8a81ed606fcd..5755bf319e8d2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java @@ -36,6 +36,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -48,6 +49,8 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.LOCAL; +import static org.apache.ignite.configuration.DataPageEvictionMode.RANDOM_2_LRU; +import static org.apache.ignite.configuration.DataPageEvictionMode.RANDOM_LRU; /** * @@ -206,6 +209,7 @@ public void testNodeRestartWithCacheModeChangedMvccToTx() throws Exception { DataStorageConfiguration storageCfg = new DataStorageConfiguration(); DataRegionConfiguration regionCfg = new DataRegionConfiguration(); regionCfg.setPersistenceEnabled(true); + regionCfg.setPageEvictionMode(RANDOM_LRU); storageCfg.setDefaultDataRegionConfiguration(regionCfg); IgniteConfiguration cfg = getConfiguration("testGrid"); cfg.setDataStorageConfiguration(storageCfg); @@ -242,6 +246,40 @@ public void testNodeRestartWithCacheModeChangedMvccToTx() throws Exception { }, IgniteCheckedException.class, "Cannot start cache. Statically configured atomicity mode"); } + /** + * @throws Exception If failed. + */ + public void testMvccInMemoryEvictionDisabled() throws Exception { + final String memRegName = "in-memory-evictions"; + + // Enable in-memory eviction. + DataRegionConfiguration regionCfg = new DataRegionConfiguration(); + regionCfg.setPersistenceEnabled(false); + regionCfg.setPageEvictionMode(RANDOM_2_LRU); + regionCfg.setName(memRegName); + + DataStorageConfiguration storageCfg = new DataStorageConfiguration(); + storageCfg.setDefaultDataRegionConfiguration(regionCfg); + + IgniteConfiguration cfg = getConfiguration("testGrid"); + cfg.setDataStorageConfiguration(storageCfg); + + Ignite node = startGrid(cfg); + + CacheConfiguration ccfg1 = new CacheConfiguration("test1") + .setAtomicityMode(TRANSACTIONAL_SNAPSHOT) + .setDataRegionName(memRegName); + + try { + node.createCache(ccfg1); + + fail("In memory evictions should be disabled for MVCC caches."); + } + catch (Exception e) { + assertTrue(X.getFullStackTrace(e).contains("Data pages evictions cannot be used with TRANSACTIONAL_SNAPSHOT")); + } + } + /** * Test TRANSACTIONAL_SNAPSHOT and near cache. * From b6c64a6e6e09fbb93b3953ac7384d9c815b8c5dd Mon Sep 17 00:00:00 2001 From: rkondakov Date: Tue, 25 Dec 2018 17:10:34 +0300 Subject: [PATCH 282/403] IGNITE-10785: MVCC: Fix TxRollbackAsyncTest. This closes #5735. (cherry picked from commit a0164b47139adb0a28ec8f4f4e5fbc401bae16e0) --- .../transactions/TxRollbackAsyncTest.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java index d742ed504655c..fe16def5a38d5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java @@ -51,7 +51,6 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteFutureCancelledCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.cache.GridCacheContext; @@ -351,14 +350,14 @@ public void testSynchronousRollback() throws Exception { */ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockNode, final boolean useTimeout) throws Exception { - final CountDownLatch keyLocked = new CountDownLatch(1); + final GridFutureAdapter keyLocked = new GridFutureAdapter<>(); CountDownLatch waitCommit = new CountDownLatch(1); // Used for passing tx instance to rollback thread. IgniteInternalFuture lockFut = lockInTx(holdLockNode, keyLocked, waitCommit, 0); - U.awaitQuiet(keyLocked); + keyLocked.get(); final int txCnt = 1000; @@ -381,8 +380,6 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN IgniteInternalFuture txFut = multithreadedAsync(new Runnable() { @Override public void run() { - U.awaitQuiet(keyLocked); - for (int i = 0; i < txCnt; i++) { GridNearTxLocal tx0 = ctx.tm().threadLocalTx(cctx); @@ -836,7 +833,7 @@ public void testMixedAsyncRollbackTypes() throws Exception { * Tests proxy object returned by {@link IgniteTransactions#localActiveTransactions()} */ public void testRollbackProxy() throws Exception { - final CountDownLatch keyLocked = new CountDownLatch(1); + final GridFutureAdapter keyLocked = new GridFutureAdapter<>(); CountDownLatch waitCommit = new CountDownLatch(1); @@ -844,7 +841,7 @@ public void testRollbackProxy() throws Exception { IgniteInternalFuture lockFut = lockInTx(ig, keyLocked, waitCommit, 0); - U.awaitQuiet(keyLocked); + keyLocked.get(); Collection txs = ig.transactions().localActiveTransactions(); @@ -1042,29 +1039,31 @@ public void testRollbackOnTopologyLockPessimistic() throws Exception { * Locks entry in tx and delays commit until signalled. * * @param node Near node. - * @param keyLocked Latch for notifying until key is locked. + * @param keyLocked Future to be done when key is locked. * @param waitCommit Latch for waiting until commit is allowed. * @param timeout Timeout. * @return tx completion future. */ - private IgniteInternalFuture lockInTx(final Ignite node, final CountDownLatch keyLocked, + private IgniteInternalFuture lockInTx(final Ignite node, final GridFutureAdapter keyLocked, final CountDownLatch waitCommit, final int timeout) throws Exception { return multithreadedAsync(new Runnable() { @Override public void run() { - Transaction tx = node.transactions().withLabel(LABEL).txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 1); + try { + Transaction tx = node.transactions().withLabel(LABEL).txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 1); - node.cache(CACHE_NAME).put(0, 0); + node.cache(CACHE_NAME).put(0, 0); - keyLocked.countDown(); + keyLocked.onDone(); - try { U.await(waitCommit); + + tx.commit(); } - catch (IgniteInterruptedCheckedException e) { - fail("Lock thread was interrupted while waiting"); - } + catch (Throwable e) { + keyLocked.onDone(e); - tx.commit(); + throw new RuntimeException(e); + } } }, 1, "tx-lock-thread"); } From 7da39d8bf77025da871c797d3469018607c5244b Mon Sep 17 00:00:00 2001 From: rkondakov Date: Tue, 25 Dec 2018 17:17:27 +0300 Subject: [PATCH 283/403] IGNITE-10474: MVCC: IgniteCacheConnectionRecovery10ConnectionsTest.testConnectionRecovery. This closes #5698. (cherry picked from commit 2dabbd2f7c362ad140f5225fa777dc889a7d0d7c) --- .../IgniteCacheConnectionRecoveryTest.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java index cc92ec13418a7..a2325e8aec4ff 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecoveryTest.java @@ -39,11 +39,13 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -75,7 +77,7 @@ public class IgniteCacheConnectionRecoveryTest extends GridCommonAbstractTest { cfg.setCacheConfiguration( cacheConfiguration("cache1", TRANSACTIONAL), - //cacheConfiguration("cache2", TRANSACTIONAL_SNAPSHOT), //TODO IGNITE-10474: add Mvcc cache after fix. + cacheConfiguration("cache2", TRANSACTIONAL_SNAPSHOT), cacheConfiguration("cache3", ATOMIC)); return cfg; @@ -119,15 +121,25 @@ public void testConnectionRecovery() throws Exception { IgniteCache[] caches = { node.cache("cache1"), -// node.cache("cache2"), //TODO IGNITE-10474: add Mvcc cache after fix. + node.cache("cache2"), node.cache("cache3")}; int iter = 0; while (U.currentTimeMillis() < stopTime) { try { - for (IgniteCache cache : caches) - cache.putAllAsync(data).get(15, SECONDS); + for (IgniteCache cache : caches) { + while (true) { + try { + cache.putAllAsync(data).get(15, SECONDS); + + break; + } + catch (Exception e) { + MvccFeatureChecker.assertMvccWriteConflict(e); + } + } + } CyclicBarrier b = barrierRef.get(); From 10ebf71557f084d05eba9b7855e75fef2eda748c Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 26 Dec 2018 16:28:37 +0300 Subject: [PATCH 284/403] IGNITE-10493 Fix exchange local stages comparator Signed-off-by: Pavel Kovalenko (cherry picked from commit 0431c28) Signed-off-by: Pavel Kovalenko --- .../main/java/org/apache/ignite/internal/util/TimeBag.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java b/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java index 0da1ee7ad4611..156d826e9935c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/TimeBag.java @@ -302,9 +302,9 @@ public long time() { /** {@inheritDoc} */ @Override public int compareTo(@NotNull TimeBag.Stage o) { - if (o.time > time) - return -1; if (o.time < time) + return -1; + if (o.time > time) return 1; return o.description.compareTo(description); } From b3b1e372c31fe1ebb88a2bbea35f4e790a367abc Mon Sep 17 00:00:00 2001 From: zstan Date: Tue, 25 Dec 2018 12:01:34 +0300 Subject: [PATCH 285/403] IGNITE-9493 Do not call communication error resolver in case of client node failed Signed-off-by: Pavel Kovalenko (cherry picked from commit 2b23d46feba75b1eb51c8373c730b20504d1b30a) --- .../discovery/zk/ZookeeperDiscoverySpi.java | 9 +- .../zk/ZookeeperDiscoverySpiMBean.java | 8 ++ .../zk/internal/ZookeeperDiscoveryImpl.java | 5 + .../ZookeeperDiscoveryStatistics.java | 21 +++- .../internal/ZookeeperDiscoverySpiTest.java | 95 ++++++++++++++++++- 5 files changed, 131 insertions(+), 7 deletions(-) diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java index 52f77fcc8a871..2b8215750bccf 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java @@ -588,12 +588,17 @@ public ZookeeperDiscoverySpiMBeanImpl(IgniteSpiAdapter spiAdapter) { } /** {@inheritDoc} */ - @Nullable @Override public UUID getCoordinator() { + @Override public long getCommErrorProcNum() { + return stats.commErrorCount(); + } + + /** {@inheritDoc} */ + @Override public @Nullable UUID getCoordinator() { return impl.getCoordinator(); } /** {@inheritDoc} */ - @Nullable @Override public String getCoordinatorNodeFormatted() { + @Override public @Nullable String getCoordinatorNodeFormatted() { return String.valueOf(impl.node(impl.getCoordinator())); } diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java index 1eed0b41d098d..05a3dc26778a3 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java @@ -57,6 +57,14 @@ public interface ZookeeperDiscoverySpiMBean extends IgniteSpiManagementMBean, Di @MXBeanDescription("Zk Session Id.") public String getZkSessionId(); + /** + * Gets number of communication resolver called. + * + * @return Number of communication resolved oparations. + */ + @MXBeanDescription("Communication error resolver call count.") + public long getCommErrorProcNum(); + /** * Gets root path in ZooKeeper cluster Zk client uses to put data to. * diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java index 2462ae60d2cdd..5e05506de77ee 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java @@ -300,6 +300,9 @@ void clearCommunicationErrorProcessFuture(ZkCommunicationErrorProcessFuture fut) * @param err Connect error. */ public void resolveCommunicationError(ClusterNode node0, Exception err) { + if (node0.isClient()) + return; + ZookeeperClusterNode node = node(node0.id()); if (node == null) @@ -317,6 +320,8 @@ public void resolveCommunicationError(ClusterNode node0, Exception err) { this, node.sessionTimeout() + 1000); + stats.onCommunicationError(); + if (commErrProcFut.compareAndSet(fut, newFut)) { fut = newFut; diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java index 678cf11cf1505..cc95dd3fe9f0e 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java @@ -23,21 +23,29 @@ */ public class ZookeeperDiscoveryStatistics { /** */ - private int joinedNodesCnt; + private long joinedNodesCnt; /** */ - private int failedNodesCnt; + private long failedNodesCnt; + + /** Communication error count. */ + private long commErrCnt; /** */ - public int joinedNodesCnt() { + public long joinedNodesCnt() { return joinedNodesCnt; } /** */ - public int failedNodesCnt() { + public long failedNodesCnt() { return failedNodesCnt; } + /** */ + public long commErrorCount() { + return commErrCnt; + } + /** */ public void onNodeJoined() { joinedNodesCnt++; @@ -48,6 +56,11 @@ public void onNodeFailed() { failedNodesCnt++; } + /** */ + public void onCommunicationError() { + commErrCnt++; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(ZookeeperDiscoveryStatistics.class, this); diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java index f249b59d0372a..b813615bb6680 100644 --- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java @@ -200,6 +200,9 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { /** */ private boolean failCommSpi; + /** */ + private boolean blockCommSpi; + /** */ private long sesTimeout; @@ -396,6 +399,13 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { if (failCommSpi) cfg.setCommunicationSpi(new PeerToPeerCommunicationFailureSpi()); + if (blockCommSpi) { + cfg.setCommunicationSpi(new TcpBlockCommunicationSpi(igniteInstanceName.contains("block")) + .setUsePairedConnections(true)); + + cfg.setNetworkTimeout(500); + } + if (commFailureRslvr != null) cfg.setCommunicationFailureResolver(commFailureRslvr.apply()); @@ -3490,6 +3500,44 @@ public void testCommunicationFailureResolve_ConcurrentMultinode() throws Excepti }, 30, "test-resolve-failure"); } + /** + * Test reproduces failure in case of client resolution failure + * {@link org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi#createTcpClient} from server side, further + * client reconnect and proper grid work. + * + * @throws Exception If failed. + */ + public void testClientReconnects() throws Exception { + blockCommSpi = true; + + Ignite srv1 = startGrid("server1-block"); + + clientModeThreadLocal(true); + + IgniteEx cli = startGrid("client-block"); + + IgniteCache cache = cli.getOrCreateCache(DEFAULT_CACHE_NAME); + + cache.put(1, 1); + + assertEquals(cache.get(1), 1); + + assertEquals(1, srv1.cluster().forClients().nodes().size()); + + MBeanServer srv = ManagementFactory.getPlatformMBeanServer(); + + IgniteEx ignite = grid("server1-block"); + + ObjectName spiName = U.makeMBeanName(ignite.context().igniteInstanceName(), "SPIs", + ZookeeperDiscoverySpi.class.getSimpleName()); + + ZookeeperDiscoverySpiMBean bean = JMX.newMBeanProxy(srv, spiName, ZookeeperDiscoverySpiMBean.class); + + assertNotNull(bean); + + assertEquals(0, bean.getCommErrorProcNum()); + } + /** * @throws Exception If failed. */ @@ -5500,7 +5548,7 @@ static class TestFastStopProcessCustomMessageAck implements DiscoveryCustomMessa } /** {@inheritDoc} */ - @Nullable @Override public DiscoveryCustomMessage ackMessage() { + @Override public @Nullable DiscoveryCustomMessage ackMessage() { return null; } @@ -5543,4 +5591,49 @@ static class TestFastStopProcessCustomMessageAck implements DiscoveryCustomMessa return S.toString(TestFastStopProcessCustomMessageAck.class, this); } } + + /** + * Block communications. + */ + private static class TcpBlockCommunicationSpi extends TcpCommunicationSpi { + /** + * Whether this instance should actually block. + */ + private final boolean isBlocking; + + /** Blocked once. */ + private boolean alreadyBlocked; + + /** + * @param isBlocking Whether this instance should actually block. + */ + public TcpBlockCommunicationSpi(boolean isBlocking) { + this.isBlocking = isBlocking; + } + + /** {@inheritDoc} */ + @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) + throws IgniteCheckedException { + if (node.isClient() && blockHandshakeOnce(node.id())) { + ZookeeperDiscoverySpi spi = spi(ignite()); + + spi.resolveCommunicationFailure(spi.getRemoteNodes().iterator().next(), new Exception("test")); + + return null; + } + + return super.createTcpClient(node, connIdx); + } + + /** Check if this connection is blocked. */ + private boolean blockHandshakeOnce(UUID nodeId) { + if (isBlocking && !alreadyBlocked) { + alreadyBlocked = true; + + return true; + } + + return false; + } + } } From 6c01717bbac5a321e2f62d8b09f293be300b451f Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Tue, 11 Dec 2018 14:19:54 +0300 Subject: [PATCH 286/403] IGNITE-10472: MVCC: EntryProcessor resource injection. Fix test. (cherry picked from commit 7633e18f1b8b6fce9aaf780c428193f91a019008) --- .../processors/cache/IgniteCacheMvccTxInvokeTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java index 7029605d8c4b5..6ed569837b22d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheMvccTxInvokeTest.java @@ -47,11 +47,4 @@ public class IgniteCacheMvccTxInvokeTest extends IgniteCacheInvokeAbstractTest { @Override protected NearCacheConfiguration nearConfiguration() { return null; } - - /** {@inheritDoc} */ - @Override public void testInvokeAllAppliedOnceOnBinaryTypeRegistration() { - fail("https://issues.apache.org/jira/browse/IGNITE-10472"); - - super.testInvokeAllAppliedOnceOnBinaryTypeRegistration(); - } } \ No newline at end of file From b2704cd9d9f5254a0a795baed813e39e204f90f2 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Thu, 27 Dec 2018 12:15:26 +0300 Subject: [PATCH 287/403] IGNITE-10167: MVCC: MVCC-compatible IgniteCache.localEntries. Unmute tests. --- .../processors/cache/IgniteCacheTxPeekModesTest.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java index b43ab9ad67266..c6c937bd4698f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheTxPeekModesTest.java @@ -19,7 +19,6 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.testframework.MvccFeatureChecker; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -42,12 +41,4 @@ public class IgniteCacheTxPeekModesTest extends IgniteCachePeekModesAbstractTest @Override protected CacheAtomicityMode atomicityMode() { return TRANSACTIONAL; } - - /** {@inheritDoc} */ - @Override public void testLocalEntries() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10167"); - - super.testLocalEntries(); - } } \ No newline at end of file From 78cf08340854e9cfc1c5602dea81ad5ebdf32ddf Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Thu, 27 Dec 2018 12:19:19 +0300 Subject: [PATCH 288/403] IGNITE-9292: MVCC: fixed a race causing unexpected state of entry in TX log. Unmute test. --- .../cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java index 62cd5b5e499e6..2b6d8bffb1b7c 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java @@ -1400,8 +1400,6 @@ public void testInsertAndFastUpdateWithoutVersionConflict() throws Exception { * @throws Exception If failed. */ public void testInsertFastUpdateConcurrent() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-9292"); - ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 2, DFLT_PARTITION_COUNT) .setIndexedTypes(Integer.class, Integer.class); From 6cc19772bfdb56b3821bea4793226f545c7d4ae3 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 27 Dec 2018 16:36:02 +0700 Subject: [PATCH 289/403] IGNITE-9845 Web Console: Added support for two way SSL between browser, web server, agent and cluster. (cherry picked from commit 9f9bb75201a3c4a4ca5398c450d741a095321167) --- .../ignite/ssl/DelegatingSSLContextSpi.java | 4 +- .../ignite/ssl/SSLSocketFactoryWrapper.java | 64 ++-- modules/web-console/assembly/README.txt | 46 ++- .../backend/app/browsersHandler.js | 34 +- modules/web-console/backend/app/settings.js | 66 +++- .../backend/config/settings.json.sample | 51 +-- modules/web-console/backend/launch-tools.js | 6 +- modules/web-console/backend/package.json | 2 +- modules/web-console/frontend/app/app.js | 3 - .../connected-clusters-badge/controller.js | 11 +- .../app/modules/agent/AgentManager.service.js | 12 +- .../frontend/app/modules/demo/Demo.module.js | 24 +- .../frontend/app/utils/SimpleWorkerPool.js | 2 +- modules/web-console/frontend/app/vendor.js | 1 - modules/web-console/frontend/package.json | 5 +- .../frontend/webpack/webpack.dev.js | 25 +- modules/web-console/web-agent/pom.xml | 18 +- .../console/agent/AgentConfiguration.java | 330 ++++++++++++++++-- .../ignite/console/agent/AgentLauncher.java | 89 ++++- .../ignite/console/agent/AgentUtils.java | 140 +++++++- .../agent/handlers/AbstractListener.java | 9 +- .../agent/handlers/ClusterListener.java | 77 ++-- .../console/agent/handlers/RestListener.java | 6 +- .../console/agent/rest/RestExecutor.java | 84 +++-- .../agent/rest/RestExecutorSelfTest.java | 329 +++++++++++++++++ .../testsuites/IgniteWebAgentTestSuite.java | 33 ++ .../web-agent/src/test/resources/ca.jks | Bin 0 -> 1394 bytes .../web-agent/src/test/resources/client.jks | Bin 0 -> 2030 bytes .../web-agent/src/test/resources/generate.bat | 122 +++++++ .../web-agent/src/test/resources/generate.sh | 111 ++++++ .../test/resources/jetty-with-ciphers-0.xml | 94 +++++ .../test/resources/jetty-with-ciphers-1.xml | 94 +++++ .../test/resources/jetty-with-ciphers-2.xml | 94 +++++ .../src/test/resources/jetty-with-ssl.xml | 89 +++++ .../web-agent/src/test/resources/server.jks | Bin 0 -> 1419 bytes 35 files changed, 1794 insertions(+), 281 deletions(-) create mode 100644 modules/web-console/web-agent/src/test/java/org/apache/ignite/console/agent/rest/RestExecutorSelfTest.java create mode 100644 modules/web-console/web-agent/src/test/java/org/apache/ignite/testsuites/IgniteWebAgentTestSuite.java create mode 100644 modules/web-console/web-agent/src/test/resources/ca.jks create mode 100644 modules/web-console/web-agent/src/test/resources/client.jks create mode 100644 modules/web-console/web-agent/src/test/resources/generate.bat create mode 100644 modules/web-console/web-agent/src/test/resources/generate.sh create mode 100644 modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-0.xml create mode 100644 modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-1.xml create mode 100644 modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-2.xml create mode 100644 modules/web-console/web-agent/src/test/resources/jetty-with-ssl.xml create mode 100644 modules/web-console/web-agent/src/test/resources/server.jks diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java b/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java index d8621f2cefd9a..360e92239c6bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java +++ b/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java @@ -31,7 +31,6 @@ /** */ class DelegatingSSLContextSpi extends SSLContextSpi { - /** */ private final SSLContext delegate; @@ -57,8 +56,7 @@ class DelegatingSSLContextSpi extends SSLContextSpi { /** {@inheritDoc} */ @Override protected SSLServerSocketFactory engineGetServerSocketFactory() { - return new SSLServerSocketFactoryWrapper(delegate.getServerSocketFactory(), - parameters); + return new SSLServerSocketFactoryWrapper(delegate.getServerSocketFactory(), parameters); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java b/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java index bfe6d0d6f4bca..992f836702d62 100644 --- a/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java +++ b/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java @@ -24,9 +24,10 @@ import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocketFactory; -/** */ +/** + * SSL socket factory that configure additional SSL params for socket. + */ class SSLSocketFactoryWrapper extends SSLSocketFactory { - /** */ private final SSLSocketFactory delegate; @@ -49,65 +50,46 @@ class SSLSocketFactoryWrapper extends SSLSocketFactory { return delegate.getSupportedCipherSuites(); } - /** {@inheritDoc} */ - @Override public Socket createSocket() throws IOException { - SSLSocket sock = (SSLSocket)delegate.createSocket(); - + /** + * Configure socket with SSL parameters. + * + * @param socket Socket to configure. + * @return Configured socket. + */ + private Socket configureSocket(Socket socket) { if (parameters != null) - sock.setSSLParameters(parameters); + ((SSLSocket)socket).setSSLParameters(parameters); - return sock; + return socket; } /** {@inheritDoc} */ - @Override public Socket createSocket(Socket sock, String host, int port, boolean autoClose) throws IOException { - SSLSocket sslSock = (SSLSocket)delegate.createSocket(sock, host, port, autoClose); - - if (parameters != null) - sslSock.setSSLParameters(parameters); + @Override public Socket createSocket() throws IOException { + return configureSocket(delegate.createSocket()); + } - return sock; + /** {@inheritDoc} */ + @Override public Socket createSocket(Socket sock, String host, int port, boolean autoClose) throws IOException { + return configureSocket(delegate.createSocket(sock, host, port, autoClose)); } /** {@inheritDoc} */ @Override public Socket createSocket(String host, int port) throws IOException { - SSLSocket sock = (SSLSocket)delegate.createSocket(host, port); - - if (parameters != null) - sock.setSSLParameters(parameters); - - return sock; + return configureSocket(delegate.createSocket(host, port)); } /** {@inheritDoc} */ @Override public Socket createSocket(String host, int port, InetAddress locAddr, int locPort) throws IOException { - SSLSocket sock = (SSLSocket)delegate.createSocket(host, port, locAddr, locPort); - - if (parameters != null) - sock.setSSLParameters(parameters); - - return sock; + return configureSocket(delegate.createSocket(host, port, locAddr, locPort)); } /** {@inheritDoc} */ @Override public Socket createSocket(InetAddress addr, int port) throws IOException { - SSLSocket sock = (SSLSocket)delegate.createSocket(addr, port); - - if (parameters != null) - sock.setSSLParameters(parameters); - - return sock; + return configureSocket(delegate.createSocket(addr, port)); } /** {@inheritDoc} */ - @Override public Socket createSocket(InetAddress addr, int port, InetAddress locAddr, - int locPort) throws IOException { - SSLSocket sock = (SSLSocket)delegate.createSocket(addr, port, locAddr, locPort); - - if (parameters != null) - sock.setSSLParameters(parameters); - - return sock; + @Override public Socket createSocket(InetAddress addr, int port, InetAddress locAddr, int locPort) throws IOException { + return configureSocket(delegate.createSocket(addr, port, locAddr, locPort)); } - } diff --git a/modules/web-console/assembly/README.txt b/modules/web-console/assembly/README.txt index f1e114c523b0a..2656aca95fd52 100644 --- a/modules/web-console/assembly/README.txt +++ b/modules/web-console/assembly/README.txt @@ -43,12 +43,13 @@ Technical details All available parameters with defaults: Web Console host: --server:host 0.0.0.0 Web Console port: --server:port 80 + Enable HTTPS: --server:ssl false - HTTPS key: --server:key "serve/keys/test.key" - HTTPS certificate: --server:cert "serve/keys/test.crt" - HTTPS passphrase: --server:keyPassphrase "password" + Disable self registration: --server:disable:signup false + MongoDB URL: --mongodb:url mongodb://localhost/console + Mail service: --mail:service "gmail" Signature text: --mail:sign "Kind regards, Apache Ignite Team" Greeting text: --mail:greeting "Apache Ignite Web Console" @@ -56,8 +57,31 @@ All available parameters with defaults: User to send e-mail: --mail:auth:user "someusername@somecompany.somedomain" E-mail service password: --mail:auth:pass "" -Sample usage: - `ignite-web-console-win.exe --mail:auth:user "my_user@gmail.com" --mail:auth:pass "my_password"` +SSL options has no default values: + --server:key "path to file with server.key" + --server:cert "path to file with server.crt" + --server:ca "path to file with ca.crt" + --server:passphrase "Password for key" + --server:ciphers "Comma separated ciphers list" + --server:secureProtocol "The TLS protocol version to use" + --server:clientCertEngine "Name of an OpenSSL engine which can provide the client certificate" + --server:pfx "Path to PFX or PKCS12 encoded private key and certificate chain" + --server:crl "Path to file with CRLs (Certificate Revocation Lists)" + --server:dhparam "Diffie Hellman parameters" + --server:ecdhCurve "A string describing a named curve" + --server:maxVersion "Optional the maximmu TLS version to allow" + --server:minVersion "Optional the minimum TLS version to allow" + --server:secureOptions "Optional OpenSSL options" + --server:sessionIdContext "Opaque identifier used by servers to ensure session state is not shared between applications" + --server:honorCipherOrder "true or false" + --server:requestCert "Set to true to specify whether a server should request a certificate from a connecting client" + --server:rejectUnauthorized "Set to true to automatically reject clients with invalid certificates" + +Documentation for SSL options: https://nodejs.org/api/tls.html#tls_tls_createsecurecontext_options + +Sample usages: + `ignite-web-console-win.exe --mail:auth:user "my_user@gmail.com" --mail:auth:pass "my_password"` + `ignite-web-console-win.exe --server:port 11443 --server:ssl true --server:requestCert true --server:key "server.key" --server:cert "server.crt" --server:ca "ca.crt" --server:passphrase "my_password"` Advanced configuration of SMTP for Web Console. ------------------------------------- @@ -90,9 +114,11 @@ In case of non GMail SMTP server it may require to change options in "settings.j Troubleshooting ------------------------------------- 1. On Windows check that MongoDB is not blocked by Antivirus/Firewall/Smartscreen. -2. Root permission is required to bind to 80 port under Mac OS X and Linux, but you may always start Web Console on another port if you don't have such permission. +2. Root permission is required to bind to 80 port under macOS and Linux, but you may always start Web Console + on another port if you don't have such permission. 3. For extended debug output start Web Console as following: - On Linux execute command in terminal: `DEBUG=mongodb-* ./ignite-web-console-linux` - On Windows execute two commands in terminal: - `SET DEBUG=mongodb-*` - `ignite-web-console-win.exe` + On Linux execute command in terminal: `DEBUG=mongodb-* ./ignite-web-console-linux` + On macOS execute command in terminal: `DEBUG=mongodb-* ./ignite-web-console-macos` + On Windows execute two commands in terminal: + `SET DEBUG=mongodb-*` + `ignite-web-console-win.exe` diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js index a7156c5281315..820c3d47272da 100644 --- a/modules/web-console/backend/app/browsersHandler.js +++ b/modules/web-console/backend/app/browsersHandler.js @@ -38,12 +38,12 @@ module.exports = { * @param {Socket} sock */ add(sock) { - const token = sock.request.user._id.toString(); + const key = sock.request.user._id.toString(); - if (this.sockets.has(token)) - this.sockets.get(token).push(sock); + if (this.sockets.has(key)) + this.sockets.get(key).push(sock); else - this.sockets.set(token, [sock]); + this.sockets.set(key, [sock]); return this.sockets.get(sock.request.user); } @@ -52,9 +52,9 @@ module.exports = { * @param {Socket} sock */ remove(sock) { - const token = sock.request.user.token; + const key = sock.request.user._id.toString(); - const sockets = this.sockets.get(token); + const sockets = this.sockets.get(key); _.pull(sockets, sock); @@ -62,17 +62,15 @@ module.exports = { } get(account) { - let sockets = this.sockets.get(account._id.toString()); + const key = account._id.toString(); + + let sockets = this.sockets.get(key); if (_.isEmpty(sockets)) - this.sockets.set(account._id.toString(), sockets = []); + this.sockets.set(key, sockets = []); return sockets; } - - demo(token) { - return _.filter(this.sockets.get(token), (sock) => sock.request._query.IgniteDemoMode === 'true'); - } } class BrowsersHandler { @@ -191,10 +189,10 @@ module.exports = { * @param {Object.} params * @return {Promise.} */ - executeOnNode(agent, demo, credentials, params) { + executeOnNode(agent, token, demo, credentials, params) { return agent .then((agentSock) => agentSock.emitEvent('node:rest', - {uri: 'ignite', demo, params: _.merge({}, credentials, params)})); + {uri: 'ignite', token, demo, params: _.merge({}, credentials, params)})); } registerVisorTask(taskId, taskCls, ...argCls) { @@ -222,7 +220,9 @@ module.exports = { const agent = this._agentHnd.agent(sock.request.user, demo, clusterId); - this.executeOnNode(agent, demo, credentials, params) + const token = sock.request.user.token; + + this.executeOnNode(agent, token, demo, credentials, params) .then((data) => cb(null, data)) .catch((err) => cb(this.errorTransformer(err))); }); @@ -282,7 +282,9 @@ module.exports = { const agent = this._agentHnd.agent(sock.request.user, demo, clusterId); - this.executeOnNode(agent, demo, credentials, exeParams) + const token = sock.request.user.token; + + this.executeOnNode(agent, token, demo, credentials, exeParams) .then((data) => { if (data.finished && !data.zipped) return cb(null, data.result); diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js index 233566b1b76d8..9c4d3c3fb380d 100644 --- a/modules/web-console/backend/app/settings.js +++ b/modules/web-console/backend/app/settings.js @@ -61,7 +61,7 @@ module.exports = { return v === 'true' || v === true; }; - return { + const settings = { agent: { dists: nconf.get('agent:dists') || dfltAgentDists }, @@ -69,15 +69,6 @@ module.exports = { server: { host: nconf.get('server:host') || dfltHost, port: _normalizePort(nconf.get('server:port') || dfltPort), - // eslint-disable-next-line eqeqeq - SSLOptions: _isTrue('server:ssl') && { - enable301Redirects: true, - trustXFPHeader: true, - key: fs.readFileSync(nconf.get('server:key')), - cert: fs.readFileSync(nconf.get('server:cert')), - passphrase: nconf.get('server:keyPassphrase') - }, - // eslint-disable-next-line eqeqeq disableSignup: _isTrue('server:disable:signup') }, mail, @@ -86,5 +77,60 @@ module.exports = { sessionSecret: nconf.get('server:sessionSecret') || 'keyboard cat', tokenLength: 20 }; + + // Configure SSL options. + if (_isTrue('server:ssl')) { + const sslOptions = { + enable301Redirects: true, + trustXFPHeader: true, + isServer: true + }; + + const setSslOption = (name, fromFile = false) => { + const v = nconf.get(`server:${name}`); + + const hasOption = !!v; + + if (hasOption) + sslOptions[name] = fromFile ? fs.readFileSync(v) : v; + + return hasOption; + }; + + const setSslOptionBoolean = (name) => { + const v = nconf.get(`server:${name}`); + + if (v) + sslOptions[name] = v === 'true' || v === true; + }; + + setSslOption('key', true); + setSslOption('cert', true); + setSslOption('ca', true); + setSslOption('passphrase'); + setSslOption('ciphers'); + setSslOption('secureProtocol'); + setSslOption('clientCertEngine'); + setSslOption('pfx', true); + setSslOption('crl'); + setSslOption('dhparam'); + setSslOption('ecdhCurve'); + setSslOption('maxVersion'); + setSslOption('minVersion'); + setSslOption('secureOptions'); + setSslOption('sessionIdContext'); + + setSslOptionBoolean('honorCipherOrder'); + setSslOptionBoolean('requestCert'); + setSslOptionBoolean('rejectUnauthorized'); + + // Special care for case, when user set password for something like "123456". + if (sslOptions.passphrase) + sslOptions.passphrase = sslOptions.passphrase.toString(); + + settings.server.SSLOptions = sslOptions; + } + + return settings; } }; diff --git a/modules/web-console/backend/config/settings.json.sample b/modules/web-console/backend/config/settings.json.sample index aa20ab46f5c92..c16ba265bd655 100644 --- a/modules/web-console/backend/config/settings.json.sample +++ b/modules/web-console/backend/config/settings.json.sample @@ -1,26 +1,31 @@ { - "server": { - "port": 3000, - "sessionSecret": "CHANGE ME", - "ssl": false, - "key": "serve/keys/test.key", - "cert": "serve/keys/test.crt", - "keyPassphrase": "password", - "disable": { - "signup": false - } - }, - "mongodb": { - "url": "mongodb://localhost/console" - }, - "mail": { - "service": "", - "from": "Some Company Web Console ", - "greeting": "Some Company Web Console", - "sign": "Kind regards,
Some Company Team", - "auth": { - "user": "some_username@some_company.com", - "pass": "" - } + "server": { + "port": 3000, + "sessionSecret": "CHANGE ME", + "ssl": false, + "key": "path to file with server.key", + "cert": "path to file with server.crt", + "ca": "path to file with ca.crt", + "passphrase": "password", + "ciphers": "ECDHE-RSA-AES128-GCM-SHA256", + "secureProtocol": "TLSv1_2_method", + "requestCert": false, + "rejectUnauthorized": false, + "disable": { + "signup": false } + }, + "mongodb": { + "url": "mongodb://localhost/console" + }, + "mail": { + "service": "gmail", + "from": "Some Company Web Console ", + "greeting": "Some Company Web Console", + "sign": "Kind regards,
Some Company Team", + "auth": { + "user": "some_username@some_company.com", + "pass": "CHANGE ME" + } + } } diff --git a/modules/web-console/backend/launch-tools.js b/modules/web-console/backend/launch-tools.js index 3870b0d6c754f..0d6114fb363a6 100644 --- a/modules/web-console/backend/launch-tools.js +++ b/modules/web-console/backend/launch-tools.js @@ -55,7 +55,11 @@ const _onError = (addr, error) => { */ const init = ([settings, apiSrv, agentsHnd, browsersHnd]) => { // Start rest server. - const srv = settings.server.SSLOptions ? https.createServer(settings.server.SSLOptions) : http.createServer(); + const sslOptions = settings.server.SSLOptions; + + console.log(`Starting ${sslOptions ? 'HTTPS' : 'HTTP'} server`); + + const srv = sslOptions ? https.createServer(sslOptions) : http.createServer(); srv.listen(settings.server.port, settings.server.host); diff --git a/modules/web-console/backend/package.json b/modules/web-console/backend/package.json index fb127489f2667..4399ae79f6547 100644 --- a/modules/web-console/backend/package.json +++ b/modules/web-console/backend/package.json @@ -68,7 +68,7 @@ "passport-local-mongoose": "4.0.0", "passport.socketio": "3.7.0", "pkg": "4.3.1", - "socket.io": "1.7.3", + "socket.io": "2.1.1", "uuid": "3.1.0" }, "devDependencies": { diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index e1cbdbb32bcf4..fd75adee0ecdd 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -44,7 +44,6 @@ import './modules/configuration/configuration.module'; import './modules/getting-started/GettingStarted.provider'; import './modules/dialog/dialog.module'; import './modules/ace.module'; -import './modules/socket.module'; import './modules/loading/loading.module'; import servicesModule from './services'; // endignite @@ -177,7 +176,6 @@ export default angular.module('ignite-console', [ 'ngSanitize', 'ngMessages', // Third party libs. - 'btford.socket-io', 'dndLists', 'gridster', 'mgcrea.ngStrap', @@ -201,7 +199,6 @@ export default angular.module('ignite-console', [ 'ignite-console.input-dialog', 'ignite-console.user', 'ignite-console.branding', - 'ignite-console.socket', 'ignite-console.agent', 'ignite-console.nodes', 'ignite-console.demo', diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js index 807c3ba8b7d65..60a06965e2a65 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js +++ b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js @@ -16,15 +16,17 @@ */ export default class { - static $inject = ['AgentManager', 'ConnectedClustersDialog']; + static $inject = ['$scope', 'AgentManager', 'ConnectedClustersDialog']; connectedClusters = 0; /** + * @param $scope Angular scope. * @param {import('app/modules/agent/AgentManager.service').default} agentMgr * @param {import('../connected-clusters-dialog/service').default} connectedClustersDialog */ - constructor(agentMgr, connectedClustersDialog) { + constructor($scope, agentMgr, connectedClustersDialog) { + this.$scope = $scope; this.agentMgr = agentMgr; this.connectedClustersDialog = connectedClustersDialog; } @@ -38,7 +40,10 @@ export default class { $onInit() { this.connectedClusters$ = this.agentMgr.connectionSbj .do(({ clusters }) => this.connectedClusters = clusters.length) - .do(({ clusters }) => this.clusters = clusters) + .do(({ clusters }) => { + this.clusters = clusters; + this.$scope.$applyAsync(); + }) .subscribe(); } diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index ef0d4237609fa..6450cd67da230 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -24,6 +24,8 @@ import 'rxjs/add/operator/partition'; import 'rxjs/add/operator/takeUntil'; import 'rxjs/add/operator/pluck'; +import io from 'socket.io-client'; + import AgentModal from './AgentModal.service'; // @ts-ignore import Worker from './decompress.worker'; @@ -120,7 +122,7 @@ class ConnectionState { } export default class AgentManager { - static $inject = ['$rootScope', '$q', '$transitions', 'igniteSocketFactory', 'AgentModal', 'UserNotifications', 'IgniteVersion', 'ClusterLoginService']; + static $inject = ['$rootScope', '$q', '$transitions', 'AgentModal', 'UserNotifications', 'IgniteVersion', 'ClusterLoginService']; /** @type {ng.IScope} */ $root; @@ -165,17 +167,15 @@ export default class AgentManager { * @param {ng.IRootScopeService} $root * @param {ng.IQService} $q * @param {import('@uirouter/angularjs').TransitionService} $transitions - * @param {unknown} socketFactory * @param {import('./AgentModal.service').default} agentModal * @param {import('app/components/user-notifications/service').default} UserNotifications * @param {import('app/services/Version.service').default} Version * @param {import('./components/cluster-login/service').default} ClusterLoginSrv */ - constructor($root, $q, $transitions, socketFactory, agentModal, UserNotifications, Version, ClusterLoginSrv) { + constructor($root, $q, $transitions, agentModal, UserNotifications, Version, ClusterLoginSrv) { this.$root = $root; this.$q = $q; this.$transitions = $transitions; - this.socketFactory = socketFactory; this.agentModal = agentModal; this.UserNotifications = UserNotifications; this.Version = Version; @@ -220,7 +220,9 @@ export default class AgentManager { if (nonNil(this.socket)) return; - this.socket = this.socketFactory(); + const options = this.isDemoMode() ? {query: 'IgniteDemoMode=true'} : {}; + + this.socket = io.connect(options); const onDisconnect = () => { const conn = this.connectionSbj.getValue(); diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js index 9416bca6627ea..57f3a022b7cf8 100644 --- a/modules/web-console/frontend/app/modules/demo/Demo.module.js +++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js @@ -25,19 +25,15 @@ const DEMO_QUERY_STATE = {state: 'base.sql.notebook', params: {noteId: 'demo'}}; /** * @param {import('@uirouter/angularjs').StateProvider} $state * @param {ng.IHttpProvider} $http - * @param {unknown} socketFactory */ -export function DemoProvider($state, $http, socketFactory) { +export function DemoProvider($state, $http) { if (/(\/demo.*)/ig.test(location.pathname)) sessionStorage.setItem('IgniteDemoMode', 'true'); const enabled = sessionStorage.getItem('IgniteDemoMode') === 'true'; - if (enabled) { - socketFactory.set({query: 'IgniteDemoMode=true'}); - + if (enabled) $http.interceptors.push('demoInterceptor'); - } function service($root) { $root.IgniteDemoMode = enabled; @@ -50,7 +46,7 @@ export function DemoProvider($state, $http, socketFactory) { return this; } -DemoProvider.$inject = ['$stateProvider', '$httpProvider', 'igniteSocketFactoryProvider']; +DemoProvider.$inject = ['$stateProvider', '$httpProvider']; /** * @param {{enabled: boolean}} Demo @@ -178,11 +174,9 @@ function config($stateProvider) { config.$inject = ['$stateProvider']; angular -.module('ignite-console.demo', [ - 'ignite-console.socket' -]) -.config(config) -.provider('Demo', DemoProvider) -.factory('demoInterceptor', demoInterceptor) -.provider('igniteDemoInfo', igniteDemoInfoProvider) -.service('DemoInfo', DemoInfo); + .module('ignite-console.demo', []) + .config(config) + .provider('Demo', DemoProvider) + .factory('demoInterceptor', demoInterceptor) + .provider('igniteDemoInfo', igniteDemoInfoProvider) + .service('DemoInfo', DemoInfo); diff --git a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js index 8e7aab74e849c..1beb418469ba8 100644 --- a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js +++ b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js @@ -90,7 +90,7 @@ export default class SimpleWorkerPool { worker.postMessage(task.data); if (this.__dbg) - console.timeEnd('Post message'); + console.timeEnd(`Post message[pool=${this._name}]`); } terminate() { diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js index 90808be43ad55..e9fc8e9567061 100644 --- a/modules/web-console/frontend/app/vendor.js +++ b/modules/web-console/frontend/app/vendor.js @@ -22,7 +22,6 @@ import 'angular-animate'; import 'angular-sanitize'; import 'angular-strap'; import 'angular-strap/dist/angular-strap.tpl'; -import 'angular-socket-io'; import 'angular-messages'; import '@uirouter/angularjs'; diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json index e8f4ae2f05320..3c4473f2c3035 100644 --- a/modules/web-console/frontend/package.json +++ b/modules/web-console/frontend/package.json @@ -44,7 +44,6 @@ "angular-nvd3": "1.0.9", "angular-sanitize": "1.7.4", "angular-smart-table": "2.1.11", - "angular-socket-io": "0.7.0", "angular-strap": "2.3.12", "angular-translate": "2.18.1", "angular-tree-control": "0.2.28", @@ -60,7 +59,7 @@ "file-saver": "1.3.3", "font-awesome": "4.7.0", "jquery": "3.2.1", - "json-bigint": "0.2.3", + "json-bigint": "0.3.0", "jsondiffpatch": "0.2.5", "jszip": "3.1.5", "lodash": "4.17.11", @@ -71,7 +70,7 @@ "resize-observer-polyfill": "1.5.0", "roboto-font": "0.1.0", "rxjs": "5.5.11", - "socket.io-client": "1.7.3", + "socket.io-client": "2.1.1", "tf-metatags": "2.0.0" }, "devDependencies": { diff --git a/modules/web-console/frontend/webpack/webpack.dev.js b/modules/web-console/frontend/webpack/webpack.dev.js index 3af63776b4189..59e3d45470dcb 100644 --- a/modules/web-console/frontend/webpack/webpack.dev.js +++ b/modules/web-console/frontend/webpack/webpack.dev.js @@ -23,9 +23,11 @@ const commonCfg = require('./webpack.common'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); -const backendPort = process.env.BACKEND_PORT || 3000; -const devServerPort = process.env.PORT || 9000; -const devServerHost = process.env.HOST || '0.0.0.0'; +const backendUrl = process.env.BACKEND_URL || 'http://localhost:3000'; +const webpackDevServerHost = process.env.HOST || '0.0.0.0'; +const webpackDevServerPort = process.env.PORT || 9000; + +console.log(`Backend url: ${backendUrl}`); module.exports = merge(commonCfg, { mode: 'development', @@ -70,15 +72,18 @@ module.exports = merge(commonCfg, { inline: true, proxy: { '/socket.io': { - target: `http://localhost:${backendPort}`, - ws: true + target: backendUrl, + ws: true, + secure: false }, '/agents': { - target: `http://localhost:${backendPort}`, - ws: true + target: backendUrl, + ws: true, + secure: false }, '/api/*': { - target: `http://localhost:${backendPort}` + target: backendUrl, + secure: false } }, watchOptions: { @@ -86,7 +91,7 @@ module.exports = merge(commonCfg, { poll: 2000 }, stats: 'errors-only', - host: devServerHost, - port: devServerPort + host: webpackDevServerHost, + port: webpackDevServerPort } }); diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml index 06778c3aad609..51bcabb0ddcc2 100644 --- a/modules/web-console/web-agent/pom.xml +++ b/modules/web-console/web-agent/pom.xml @@ -43,7 +43,7 @@ io.socket socket.io-client - 0.8.3 + 1.0.0 @@ -55,13 +55,13 @@ com.beust jcommander - 1.48 + 1.58 com.squareup.okhttp3 okhttp - 3.7.0 + 3.12.0 @@ -112,6 +112,18 @@ ignite-web-agent-${project.version} + + + src/test/java + + **/*.java + + + + src/test/resources + + + maven-jar-plugin diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java index bb2a8a2a49f13..1a919d00a40b8 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java @@ -17,7 +17,6 @@ package org.apache.ignite.console.agent; -import com.beust.jcommander.Parameter; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; @@ -28,6 +27,8 @@ import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.stream.Collectors; +import com.beust.jcommander.Parameter; import org.apache.ignite.internal.util.typedef.F; import static java.nio.charset.StandardCharsets.UTF_8; @@ -95,6 +96,51 @@ public class AgentConfiguration { " Default value: false") private Boolean disableDemo; + /** */ + @Parameter(names = {"-nks", "--node-key-store"}, + description = "Path to key store that will be used to connect to cluster") + private String nodeKeyStore; + + /** */ + @Parameter(names = {"-nkp", "--node-key-store-password"}, + description = "Optional password for node key store") + private String nodeKeyStorePass; + + /** */ + @Parameter(names = {"-nts", "--node-trust-store"}, + description = "Path to trust store that will be used to connect to cluster") + private String nodeTrustStore; + + /** */ + @Parameter(names = {"-ntp", "--node-trust-store-password"}, + description = "Optional password for node trust store") + private String nodeTrustStorePass; + + /** */ + @Parameter(names = {"-sks", "--server-key-store"}, + description = "Path to key store that will be used to connect to Web server") + private String srvKeyStore; + + /** */ + @Parameter(names = {"-skp", "--server-key-store-password"}, + description = "Optional password for server key store") + private String srvKeyStorePass; + + /** */ + @Parameter(names = {"-sts", "--server-trust-store"}, + description = "Path to trust store that will be used to connect to Web server") + private String srvTrustStore; + + /** */ + @Parameter(names = {"-stp", "--server-trust-store-password"}, + description = "Optional password for server trust store") + private String srvTrustStorePass; + + /** */ + @Parameter(names = {"-cs", "--cipher-suites"}, + description = "Optional comma-separated list of SSL cipher suites to be used to connect to server and cluster") + private List cipherSuites; + /** */ @Parameter(names = {"-h", "--help"}, help = true, description = "Print this help message") private Boolean help; @@ -218,6 +264,132 @@ public void disableDemo(Boolean disableDemo) { this.disableDemo = disableDemo; } + /** + * @return Path to node key store. + */ + public String nodeKeyStore() { + return nodeKeyStore; + } + + /** + * @param nodeKeyStore Path to node key store. + */ + public void nodeKeyStore(String nodeKeyStore) { + this.nodeKeyStore = nodeKeyStore; + } + + /** + * @return Node key store password. + */ + public String nodeKeyStorePassword() { + return nodeKeyStorePass; + } + + /** + * @param nodeKeyStorePass Node key store password. + */ + public void nodeKeyStorePassword(String nodeKeyStorePass) { + this.nodeKeyStorePass = nodeKeyStorePass; + } + + /** + * @return Path to node trust store. + */ + public String nodeTrustStore() { + return nodeTrustStore; + } + + /** + * @param nodeTrustStore Path to node trust store. + */ + public void nodeTrustStore(String nodeTrustStore) { + this.nodeTrustStore = nodeTrustStore; + } + + /** + * @return Node trust store password. + */ + public String nodeTrustStorePassword() { + return nodeTrustStorePass; + } + + /** + * @param nodeTrustStorePass Node trust store password. + */ + public void nodeTrustStorePassword(String nodeTrustStorePass) { + this.nodeTrustStorePass = nodeTrustStorePass; + } + + /** + * @return Path to server key store. + */ + public String serverKeyStore() { + return srvKeyStore; + } + + /** + * @param srvKeyStore Path to server key store. + */ + public void serverKeyStore(String srvKeyStore) { + this.srvKeyStore = srvKeyStore; + } + + /** + * @return Server key store password. + */ + public String serverKeyStorePassword() { + return srvKeyStorePass; + } + + /** + * @param srvKeyStorePass Server key store password. + */ + public void serverKeyStorePassword(String srvKeyStorePass) { + this.srvKeyStorePass = srvKeyStorePass; + } + + /** + * @return Path to server trust store. + */ + public String serverTrustStore() { + return srvTrustStore; + } + + /** + * @param srvTrustStore Path to server trust store. + */ + public void serverTrustStore(String srvTrustStore) { + this.srvTrustStore = srvTrustStore; + } + + /** + * @return Server trust store password. + */ + public String serverTrustStorePassword() { + return srvTrustStorePass; + } + + /** + * @param srvTrustStorePass Server trust store password. + */ + public void serverTrustStorePassword(String srvTrustStorePass) { + this.srvTrustStorePass = srvTrustStorePass; + } + + /** + * @return SSL cipher suites. + */ + public List cipherSuites() { + return cipherSuites; + } + + /** + * @param cipherSuites SSL cipher suites. + */ + public void cipherSuites(List cipherSuites) { + this.cipherSuites = cipherSuites; + } + /** * @return {@code true} If agent options usage should be printed. */ @@ -235,35 +407,81 @@ public void load(URL cfgUrl) throws IOException { props.load(reader); } - String val = (String)props.remove("tokens"); + String val = props.getProperty("tokens"); if (val != null) tokens(new ArrayList<>(Arrays.asList(val.split(",")))); - val = (String)props.remove("server-uri"); + val = props.getProperty("server-uri"); if (val != null) serverUri(val); - val = (String)props.remove("node-uri"); + val = props.getProperty("node-uri"); + // Intentionaly wrapped by ArrayList, for further maniulations. if (val != null) nodeURIs(new ArrayList<>(Arrays.asList(val.split(",")))); - val = (String)props.remove("node-login"); + val = props.getProperty("node-login"); if (val != null) nodeLogin(val); - val = (String)props.remove("node-password"); + val = props.getProperty("node-password"); if (val != null) nodePassword(val); - val = (String)props.remove("driver-folder"); + val = props.getProperty("driver-folder"); if (val != null) driversFolder(val); + + val = props.getProperty("node-key-store"); + + if (val != null) + nodeKeyStore(val); + + val = props.getProperty("node-key-store-password"); + + if (val != null) + nodeKeyStorePassword(val); + + val = props.getProperty("node-trust-store"); + + if (val != null) + nodeTrustStore(val); + + val = props.getProperty("node-trust-store-password"); + + if (val != null) + nodeTrustStorePassword(val); + + val = props.getProperty("server-key-store"); + + if (val != null) + serverKeyStore(val); + + val = props.getProperty("server-key-store-password"); + + if (val != null) + serverKeyStorePassword(val); + + val = props.getProperty("server-trust-store"); + + if (val != null) + serverTrustStore(val); + + val = props.getProperty("server-trust-store-password"); + + if (val != null) + serverTrustStorePassword(val); + + val = props.getProperty("cipher-suites"); + + if (val != null) + cipherSuites(Arrays.asList(val.split(","))); } /** @@ -296,43 +514,66 @@ public void merge(AgentConfiguration cfg) { if (disableDemo == null) disableDemo(cfg.disableDemo()); + + if (nodeKeyStore == null) + nodeKeyStore(cfg.nodeKeyStore()); + + if (nodeKeyStorePass == null) + nodeKeyStorePassword(cfg.nodeKeyStorePassword()); + + if (nodeTrustStore == null) + nodeTrustStore(cfg.nodeTrustStore()); + + if (nodeTrustStorePass == null) + nodeTrustStorePassword(cfg.nodeTrustStorePassword()); + + if (srvKeyStore == null) + serverKeyStore(cfg.serverKeyStore()); + + if (srvKeyStorePass == null) + serverKeyStorePassword(cfg.serverKeyStorePassword()); + + if (srvTrustStore == null) + serverTrustStore(cfg.serverTrustStore()); + + if (srvTrustStorePass == null) + serverTrustStorePassword(cfg.serverTrustStorePassword()); + + if (cipherSuites == null) + cipherSuites(cfg.cipherSuites()); + } + + /** + * @param s String with sensitive data. + * @return Secured string. + */ + private String secured(String s) { + int len = s.length(); + int toShow = len > 4 ? 4 : 1; + + return new String(new char[len - toShow]).replace('\0', '*') + s.substring(len - toShow, len); } /** {@inheritDoc} */ @Override public String toString() { StringBuilder sb = new StringBuilder(); + String nl = System.lineSeparator(); + if (!F.isEmpty(tokens)) { sb.append("User's security tokens : "); - boolean first = true; - - for (String tok : tokens) { - if (first) - first = false; - else - sb.append(','); - - if (tok.length() > 4) { - sb.append(new String(new char[tok.length() - 4]).replace('\0', '*')); - - sb.append(tok.substring(tok.length() - 4)); - } - else - sb.append(new String(new char[tok.length()]).replace('\0', '*')); - } - - sb.append('\n'); + sb.append(tokens.stream().map(this::secured).collect(Collectors.joining(", "))).append(nl); } sb.append("URI to Ignite node REST server : ") - .append(nodeURIs == null ? DFLT_NODE_URI : String.join(", ", nodeURIs)).append('\n'); + .append(nodeURIs == null ? DFLT_NODE_URI : String.join(", ", nodeURIs)).append(nl); if (nodeLogin != null) - sb.append("Login to Ignite node REST server: ").append(nodeLogin).append('\n'); + sb.append("Login to Ignite node REST server: ").append(nodeLogin).append(nl); - sb.append("URI to Ignite Console server : ").append(srvUri == null ? DFLT_SERVER_URI : srvUri).append('\n'); - sb.append("Path to agent property file : ").append(configPath()).append('\n'); + sb.append("URI to Ignite Console server : ").append(srvUri == null ? DFLT_SERVER_URI : srvUri).append(nl); + sb.append("Path to agent property file : ").append(configPath()).append(nl); String drvFld = driversFolder(); @@ -343,8 +584,35 @@ public void merge(AgentConfiguration cfg) { drvFld = new File(agentHome, "jdbc-drivers").getPath(); } - sb.append("Path to JDBC drivers folder : ").append(drvFld).append('\n'); - sb.append("Demo mode : ").append(disableDemo() ? "disabled" : "enabled"); + sb.append("Path to JDBC drivers folder : ").append(drvFld).append(nl); + sb.append("Demo mode : ").append(disableDemo() ? "disabled" : "enabled").append(nl); + + if (!F.isEmpty(nodeKeyStore)) + sb.append("Node key store : ").append(nodeKeyStore).append(nl); + + if (!F.isEmpty(nodeKeyStorePass)) + sb.append("Node key store password : ").append(secured(nodeKeyStorePass)).append(nl); + + if (!F.isEmpty(nodeTrustStore)) + sb.append("Node trust store : ").append(nodeTrustStore).append(nl); + + if (!F.isEmpty(nodeTrustStorePass)) + sb.append("Node trust store password : ").append(secured(nodeTrustStorePass)).append(nl); + + if (!F.isEmpty(srvKeyStore)) + sb.append("Server key store : ").append(srvKeyStore).append(nl); + + if (!F.isEmpty(srvKeyStorePass)) + sb.append("Server key store password : ").append(secured(srvKeyStorePass)).append(nl); + + if (!F.isEmpty(srvTrustStore)) + sb.append("Server trust store : ").append(srvTrustStore).append(nl); + + if (!F.isEmpty(srvTrustStorePass)) + sb.append("Server trust store password : ").append(secured(srvTrustStorePass)).append(nl); + + if (!F.isEmpty(cipherSuites)) + sb.append("Cipher suites : ").append(String.join(", ", cipherSuites)).append(nl); return sb.toString(); } diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java index 579f236536e95..9553aaca77bd5 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java @@ -17,12 +17,6 @@ package org.apache.ignite.console.agent; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.ParameterException; -import io.socket.client.Ack; -import io.socket.client.IO; -import io.socket.client.Socket; -import io.socket.emitter.Emitter; import java.io.File; import java.io.IOException; import java.net.Authenticator; @@ -40,9 +34,16 @@ import java.util.concurrent.CountDownLatch; import java.util.jar.Attributes; import java.util.jar.Manifest; -import javax.net.ssl.SSLContext; import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.TrustManager; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.X509TrustManager; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import io.socket.client.Ack; +import io.socket.client.IO; +import io.socket.client.Socket; +import io.socket.emitter.Emitter; +import okhttp3.OkHttpClient; import org.apache.ignite.console.agent.handlers.ClusterListener; import org.apache.ignite.console.agent.handlers.DatabaseListener; import org.apache.ignite.console.agent.handlers.RestListener; @@ -61,6 +62,8 @@ import static io.socket.client.Socket.EVENT_DISCONNECT; import static io.socket.client.Socket.EVENT_ERROR; import static org.apache.ignite.console.agent.AgentUtils.fromJSON; +import static org.apache.ignite.console.agent.AgentUtils.sslConnectionSpec; +import static org.apache.ignite.console.agent.AgentUtils.sslSocketFactory; import static org.apache.ignite.console.agent.AgentUtils.toJSON; import static org.apache.ignite.console.agent.AgentUtils.trustManager; @@ -315,28 +318,75 @@ public static void main(String[] args) throws Exception { return; } + boolean trustAll = Boolean.getBoolean("trust.all"); + boolean hasServerTrustStore = cfg.serverTrustStore() != null; + boolean hasNodeTrustStore = cfg.nodeTrustStore() != null; + + if (trustAll && hasServerTrustStore) { + log.warn("Options contains both '--server-trust-store' and '-Dtrust.all=true'. " + + "Option '-Dtrust.all=true' will be ignored."); + + trustAll = false; + } + + if (trustAll && hasNodeTrustStore) { + log.warn("Options contains both '--node-trust-store' and '-Dtrust.all=true'. " + + "Option '-Dtrust.all=true' will be ignored."); + + trustAll = false; + } + cfg.nodeURIs(nodeURIs); IO.Options opts = new IO.Options(); - opts.path = "/agents"; - // Workaround for use self-signed certificate - if (Boolean.getBoolean("trust.all")) { - log.info("Trust to all certificates mode is enabled."); - - SSLContext ctx = SSLContext.getInstance("TLS"); + List cipherSuites = cfg.cipherSuites(); + + if ( + trustAll || + hasServerTrustStore || + cfg.serverKeyStore() != null + ) { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + + X509TrustManager serverTrustMgr = trustManager( + trustAll, + cfg.serverTrustStore(), + cfg.serverTrustStorePassword() + ); + + SSLSocketFactory sslSocketFactory = sslSocketFactory( + cfg.serverKeyStore(), + cfg.serverKeyStorePassword(), + serverTrustMgr, + cipherSuites + ); + + if (sslSocketFactory != null) { + builder.sslSocketFactory(sslSocketFactory, serverTrustMgr); + + if (!F.isEmpty(cipherSuites)) + builder.connectionSpecs(sslConnectionSpec(cipherSuites)); + } - // Create an SSLContext that uses our TrustManager - ctx.init(null, new TrustManager[] {trustManager()}, null); + OkHttpClient sslFactory = builder.build(); - opts.sslContext = ctx; + opts.callFactory = sslFactory; + opts.webSocketFactory = sslFactory; + opts.secure = true; } final Socket client = IO.socket(uri, opts); - try (RestExecutor restExecutor = new RestExecutor(); - ClusterListener clusterLsnr = new ClusterListener(cfg, client, restExecutor)) { + try ( + RestExecutor restExecutor = new RestExecutor( + cfg.nodeKeyStore(), cfg.nodeKeyStorePassword(), + cfg.nodeTrustStore(), cfg.nodeTrustStorePassword(), + cipherSuites); + + ClusterListener clusterLsnr = new ClusterListener(cfg, client, restExecutor) + ) { Emitter.Listener onConnect = connectRes -> { log.info("Connection established."); @@ -416,6 +466,7 @@ public static void main(String[] args) throws Exception { }; DatabaseListener dbHnd = new DatabaseListener(cfg); + RestListener restHnd = new RestListener(cfg, restExecutor); final CountDownLatch latch = new CountDownLatch(1); diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java index 38edd976850e5..2242eb10e6890 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java @@ -17,16 +17,32 @@ package org.apache.ignite.console.agent; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule; -import io.socket.client.Ack; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; import java.security.ProtectionDomain; import java.security.cert.X509Certificate; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule; +import io.socket.client.Ack; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; +import okhttp3.ConnectionSpec; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.ssl.SSLContextWrapper; import org.apache.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; @@ -38,6 +54,9 @@ public class AgentUtils { /** */ private static final Logger log = Logger.getLogger(AgentUtils.class.getName()); + /** */ + private static final char[] EMPTY_PWD = new char[0]; + /** JSON object mapper. */ private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -50,7 +69,7 @@ public class AgentUtils { private static final Ack NOOP_CB = new Ack() { @Override public void call(Object... args) { if (args != null && args.length > 0 && args[0] instanceof Throwable) - log.error("Failed to execute request on agent.", (Throwable) args[0]); + log.error("Failed to execute request on agent.", (Throwable)args[0]); else log.info("Request on agent successfully executed " + Arrays.toString(args)); } @@ -186,13 +205,120 @@ public static T fromJSON(Object obj, Class toValType) throws IllegalArgum } /** - * Create a trust manager that trusts all certificates It is not using a particular keyStore + * @param pathToJks Path to java key store file. + * @param pwd Key store password. + * @return Key store. + * @throws GeneralSecurityException If failed to load key store. + * @throws IOException If failed to load key store file content. + */ + private static KeyStore keyStore(String pathToJks, char[] pwd) throws GeneralSecurityException, IOException { + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(new FileInputStream(pathToJks), pwd); + + return keyStore; + } + + /** + * @param keyStorePath Path to key store. + * @param keyStorePwd Key store password. + * @return Key managers. + * @throws GeneralSecurityException If failed to load key store. + * @throws IOException If failed to load key store file content. + */ + private static KeyManager[] keyManagers(String keyStorePath, String keyStorePwd) + throws GeneralSecurityException, IOException { + if (keyStorePath == null) + return null; + + char[] keyPwd = keyStorePwd != null ? keyStorePwd.toCharArray() : EMPTY_PWD; + + KeyStore keyStore = keyStore(keyStorePath, keyPwd); + + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(keyStore, keyPwd); + + return kmf.getKeyManagers(); + } + + /** + * @param trustAll {@code true} If we trust to self-signed sertificates. + * @param trustStorePath Path to trust store file. + * @param trustStorePwd Trust store password. + * @return Trust manager + * @throws GeneralSecurityException If failed to load trust store. + * @throws IOException If failed to load trust store file content. + */ + public static X509TrustManager trustManager(boolean trustAll, String trustStorePath, String trustStorePwd) + throws GeneralSecurityException, IOException { + if (trustAll) + return disabledTrustManager(); + + if (trustStorePath == null) + return null; + + char[] trustPwd = trustStorePwd != null ? trustStorePwd.toCharArray() : EMPTY_PWD; + KeyStore trustKeyStore = keyStore(trustStorePath, trustPwd); + + TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); + tmf.init(trustKeyStore); + + TrustManager[] trustMgrs = tmf.getTrustManagers(); + + return (X509TrustManager)Arrays.stream(trustMgrs) + .filter(tm -> tm instanceof X509TrustManager) + .findFirst() + .orElseThrow(() -> new IllegalStateException("X509TrustManager manager not found")); + } + + /** + * Create SSL socket factory. + * + * @param keyStorePath Path to key store. + * @param keyStorePwd Key store password. + * @param trustMgr Trust manager. + * @param cipherSuites Optional cipher suites. + * @throws GeneralSecurityException If failed to load trust store. + * @throws IOException If failed to load store file content. + */ + public static SSLSocketFactory sslSocketFactory( + String keyStorePath, String keyStorePwd, + X509TrustManager trustMgr, + List cipherSuites + ) throws GeneralSecurityException, IOException { + KeyManager[] keyMgrs = keyManagers(keyStorePath, keyStorePwd); + + if (keyMgrs == null && trustMgr == null) + return null; + + SSLContext ctx = SSLContext.getInstance("TLS"); + + if (!F.isEmpty(cipherSuites)) + ctx = new SSLContextWrapper(ctx, new SSLParameters(cipherSuites.toArray(new String[0]))); + + ctx.init(keyMgrs, new TrustManager[] {trustMgr}, null); + + return ctx.getSocketFactory(); + } + + /** + * Create SSL configuration. + * + * @param cipherSuites SSL cipher suites. + */ + public static List sslConnectionSpec(List cipherSuites) { + return Collections.singletonList(new ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS) + .cipherSuites(cipherSuites.toArray(new String[0])) + .build()); + } + + /** + * Create a trust manager that trusts all certificates. */ - public static X509TrustManager trustManager() { + private static X509TrustManager disabledTrustManager() { return new X509TrustManager() { /** {@inheritDoc} */ @Override public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; + return null; } /** {@inheritDoc} */ diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java index 33e4c2ba3becb..e0cd5953af485 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java @@ -95,12 +95,17 @@ else if (args.length == 1) } cb.call(null, toJSON(res)); - } catch (Exception e) { + } + catch (Throwable e) { + log.error("Failed to process event in pool", e); + cb.call(e, null); } }); } - catch (Exception e) { + catch (Throwable e) { + log.error("Failed to process event", e); + cb.call(e, null); } } diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java index 14d3d5d7835c8..a9f15798ce0b9 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java @@ -92,8 +92,8 @@ public class ClusterListener implements AutoCloseable { /** */ private static final String EVENT_CLUSTER_DISCONNECTED = "cluster:disconnected"; - /** Default timeout. */ - private static final long DFLT_TIMEOUT = 3000L; + /** Topology refresh frequency. */ + private static final long REFRESH_FREQ = 3000L; /** JSON object mapper. */ private static final ObjectMapper MAPPER = new GridJettyObjectMapper(); @@ -116,13 +116,13 @@ public class ClusterListener implements AutoCloseable { }; /** */ - private AgentConfiguration cfg; + private final AgentConfiguration cfg; /** */ - private Socket client; + private final Socket client; /** */ - private RestExecutor restExecutor; + private final RestExecutor restExecutor; /** */ private static final ScheduledExecutorService pool = Executors.newScheduledThreadPool(1); @@ -132,7 +132,7 @@ public class ClusterListener implements AutoCloseable { /** * @param client Client. - * @param restExecutor Client. + * @param restExecutor REST executor. */ public ClusterListener(AgentConfiguration cfg, Socket client, RestExecutor restExecutor) { this.cfg = cfg; @@ -179,7 +179,7 @@ private void safeStopRefresh() { public void watch() { safeStopRefresh(); - refreshTask = pool.scheduleWithFixedDelay(watchTask, 0L, DFLT_TIMEOUT, TimeUnit.MILLISECONDS); + refreshTask = pool.scheduleWithFixedDelay(watchTask, 0L, REFRESH_FREQ, TimeUnit.MILLISECONDS); } /** {@inheritDoc} */ @@ -426,14 +426,15 @@ else if (!F.isEmpty(cfg.nodeLogin()) && !F.isEmpty(cfg.nodePassword())) { /** * Collect topology. * - * @param full Full. + * @return REST result. + * @throws IOException If failed to collect topology. */ - private RestResult topology(boolean full) throws IOException { - Map params = U.newHashMap(3); + private RestResult topology() throws IOException { + Map params = U.newHashMap(4); params.put("cmd", "top"); params.put("attr", true); - params.put("mtr", full); + params.put("mtr", false); params.put("caches", false); return restCommand(params); @@ -476,57 +477,47 @@ public boolean active(IgniteProductVersion ver, UUID nid) throws IOException { RestResult res = restCommand(params); - switch (res.getStatus()) { - case STATUS_SUCCESS: - if (v23) - return Boolean.valueOf(res.getData()); - - return res.getData().contains("\"active\":true"); + if (res.getStatus() == STATUS_SUCCESS) + return v23 ? Boolean.valueOf(res.getData()) : res.getData().contains("\"active\":true"); - default: - throw new IOException(res.getError()); - } + throw new IOException(res.getError()); } - /** {@inheritDoc} */ @Override public void run() { try { - RestResult res = topology(false); - - switch (res.getStatus()) { - case STATUS_SUCCESS: - List nodes = MAPPER.readValue(res.getData(), - new TypeReference>() {}); - - TopologySnapshot newTop = new TopologySnapshot(nodes); + RestResult res = topology(); - if (newTop.differentCluster(top)) - log.info("Connection successfully established to cluster with nodes: " + newTop.nid8()); - else if (newTop.topologyChanged(top)) - log.info("Cluster topology changed, new topology: " + newTop.nid8()); + if (res.getStatus() == STATUS_SUCCESS) { + List nodes = MAPPER.readValue(res.getData(), + new TypeReference>() {}); - boolean active = active(newTop.clusterVersion(), F.first(newTop.getNids())); + TopologySnapshot newTop = new TopologySnapshot(nodes); - newTop.setActive(active); - newTop.setSecured(!F.isEmpty(res.getSessionToken())); + if (newTop.differentCluster(top)) + log.info("Connection successfully established to cluster with nodes: " + newTop.nid8()); + else if (newTop.topologyChanged(top)) + log.info("Cluster topology changed, new topology: " + newTop.nid8()); - top = newTop; + boolean active = active(newTop.clusterVersion(), F.first(newTop.getNids())); - client.emit(EVENT_CLUSTER_TOPOLOGY, toJSON(top)); + newTop.setActive(active); + newTop.setSecured(!F.isEmpty(res.getSessionToken())); - break; + top = newTop; - default: - LT.warn(log, res.getError()); + client.emit(EVENT_CLUSTER_TOPOLOGY, toJSON(top)); + } + else { + LT.warn(log, res.getError()); - clusterDisconnect(); + clusterDisconnect(); } } catch (ConnectException ignored) { clusterDisconnect(); } - catch (Exception e) { + catch (Throwable e) { log.error("WatchTask failed", e); clusterDisconnect(); diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java index 24c2097531c6a..0db8904da234b 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java @@ -24,6 +24,7 @@ import org.apache.ignite.console.agent.rest.RestExecutor; import org.apache.ignite.console.agent.rest.RestResult; import org.apache.ignite.console.demo.AgentClusterDemo; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; /** @@ -38,7 +39,7 @@ public class RestListener extends AbstractListener { /** * @param cfg Config. - * @param restExecutor Executor. + * @param restExecutor REST executor. */ public RestListener(AgentConfiguration cfg, RestExecutor restExecutor) { this.cfg = cfg; @@ -65,6 +66,9 @@ public RestListener(AgentConfiguration cfg, RestExecutor restExecutor) { boolean demo = (boolean)args.get("demo"); + if (F.isEmpty((String)args.get("token"))) + return RestResult.fail(401, "Request does not contain user token."); + Map headers = null; if (args.containsKey("headers")) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java index d3bdcdd6e1b87..b452b2c0a4184 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java @@ -20,11 +20,10 @@ import java.io.IOException; import java.io.StringWriter; import java.net.ConnectException; +import java.security.GeneralSecurityException; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; @@ -33,6 +32,8 @@ import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.X509TrustManager; import okhttp3.Dispatcher; import okhttp3.FormBody; import okhttp3.HttpUrl; @@ -41,6 +42,7 @@ import okhttp3.Response; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.logger.slf4j.Slf4jLogger; @@ -49,6 +51,8 @@ import static com.fasterxml.jackson.core.JsonToken.END_ARRAY; import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; +import static org.apache.ignite.console.agent.AgentUtils.sslConnectionSpec; +import static org.apache.ignite.console.agent.AgentUtils.sslSocketFactory; import static org.apache.ignite.console.agent.AgentUtils.trustManager; import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_AUTH_FAILED; import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED; @@ -68,14 +72,28 @@ public class RestExecutor implements AutoCloseable { private final OkHttpClient httpClient; /** Index of alive node URI. */ - private Map, Integer> startIdxs = U.newHashMap(2); + private final Map, Integer> startIdxs = U.newHashMap(2); /** - * Default constructor. + * Constructor. + * + * @param keyStorePath Optional path to key store file. + * @param keyStorePwd Optional password for key store. + * @param trustStorePath Optional path to trust store file. + * @param trustStorePwd Optional password for trust store. + * @param cipherSuites Optional cipher suites. + * @throws GeneralSecurityException If failed to initialize SSL. + * @throws IOException If failed to load content of key stores. */ - public RestExecutor() { + public RestExecutor( + String keyStorePath, + String keyStorePwd, + String trustStorePath, + String trustStorePwd, + List cipherSuites + ) throws GeneralSecurityException, IOException { Dispatcher dispatcher = new Dispatcher(); - + dispatcher.setMaxRequests(Integer.MAX_VALUE); dispatcher.setMaxRequestsPerHost(Integer.MAX_VALUE); @@ -83,20 +101,19 @@ public RestExecutor() { .readTimeout(0, TimeUnit.MILLISECONDS) .dispatcher(dispatcher); - // Workaround for use self-signed certificate - if (Boolean.getBoolean("trust.all")) { - try { - SSLContext ctx = SSLContext.getInstance("TLS"); + X509TrustManager trustMgr = trustManager(Boolean.getBoolean("trust.all"), trustStorePath, trustStorePwd); - // Create an SSLContext that uses our TrustManager - ctx.init(null, new TrustManager[] {trustManager()}, null); + SSLSocketFactory sslSocketFactory = sslSocketFactory( + keyStorePath, keyStorePwd, + trustMgr, + cipherSuites + ); - builder.sslSocketFactory(ctx.getSocketFactory(), trustManager()); + if (sslSocketFactory != null) { + builder.sslSocketFactory(sslSocketFactory, trustMgr); - builder.hostnameVerifier((hostname, session) -> true); - } catch (Exception ignored) { - LT.warn(log, "Failed to initialize the Trust Manager for \"-Dtrust.all\" option to skip certificate validation."); - } + if (!F.isEmpty(cipherSuites)) + builder.connectionSpecs(sslConnectionSpec(cipherSuites)); } httpClient = builder.build(); @@ -120,13 +137,10 @@ private RestResult parseResponse(Response res) throws IOException { int status = holder.getSuccessStatus(); - switch (status) { - case STATUS_SUCCESS: - return RestResult.success(holder.getResponse(), holder.getSessionToken()); + if (status == STATUS_SUCCESS) + return RestResult.success(holder.getResponse(), holder.getSessionToken()); - default: - return RestResult.fail(status, holder.getError()); - } + return RestResult.fail(status, holder.getError()); } if (res.code() == 401) @@ -171,8 +185,20 @@ private RestResult sendRequest(String url, Map params, Map nodeURIs, Map params, Map headers) throws IOException { + /** + * Send request to cluster. + * + * @param nodeURIs List of cluster nodes URIs. + * @param params Map with reques params. + * @param headers Map with reques headers. + * @return Response from cluster. + * @throws IOException If failed to send request to cluster. + */ + public RestResult sendRequest( + List nodeURIs, + Map params, + Map headers + ) throws IOException { Integer startIdx = startIdxs.getOrDefault(nodeURIs, 0); int urlsCnt = nodeURIs.size(); @@ -196,7 +222,7 @@ public RestResult sendRequest(List nodeURIs, Map params, return res; } catch (ConnectException ignored) { - LT.warn(log, "Failed to connect to cluster [url=" + nodeUrl + "]"); + LT.warn(log, "Failed connect to cluster [url=" + nodeUrl + "]"); } } @@ -274,10 +300,10 @@ public String getSessionToken() { } /** - * @param sesTokStr String representation of session token. + * @param sesTok String representation of session token. */ - public void setSessionToken(String sesTokStr) { - this.sesTok = sesTokStr; + public void setSessionToken(String sesTok) { + this.sesTok = sesTok; } } diff --git a/modules/web-console/web-agent/src/test/java/org/apache/ignite/console/agent/rest/RestExecutorSelfTest.java b/modules/web-console/web-agent/src/test/java/org/apache/ignite/console/agent/rest/RestExecutorSelfTest.java new file mode 100644 index 0000000000000..6a4fe6c22d870 --- /dev/null +++ b/modules/web-console/web-agent/src/test/java/org/apache/ignite/console/agent/rest/RestExecutorSelfTest.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.console.agent.rest; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.net.ssl.SSLHandshakeException; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.ConnectorConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Test for RestExecutor. + */ +public class RestExecutorSelfTest { + /** Name of the cache created by default in the cluster. */ + private static final String DEFAULT_CACHE_NAME = "default"; + + /** Path to certificates and configs. */ + private static final String PATH_TO_RESOURCES = "modules/web-console/web-agent/src/test/resources/"; + + /** JSON object mapper. */ + private static final ObjectMapper MAPPER = new GridJettyObjectMapper(); + + /** */ + private static final String HTTP_URI = "http://localhost:8080"; + + /** */ + private static final String HTTPS_URI = "https://localhost:8080"; + + /** */ + private static final String JETTY_WITH_SSL = "jetty-with-ssl.xml"; + + /** */ + private static final String JETTY_WITH_CIPHERS_0 = "jetty-with-ciphers-0.xml"; + + /** */ + private static final String JETTY_WITH_CIPHERS_1 = "jetty-with-ciphers-1.xml"; + + /** */ + private static final String JETTY_WITH_CIPHERS_2 = "jetty-with-ciphers-2.xml"; + + /** This cipher is disabled by default in JDK 8. */ + private static final List CIPHER_0 = Collections.singletonList("TLS_DH_anon_WITH_AES_256_GCM_SHA384"); + + /** */ + private static final List CIPHER_1 = Collections.singletonList("TLS_RSA_WITH_NULL_SHA256"); + + /** */ + private static final List CIPHER_2 = Collections.singletonList("TLS_ECDHE_ECDSA_WITH_NULL_SHA"); + + /** */ + private static final List COMMON_CIPHERS = Arrays.asList( + "TLS_RSA_WITH_NULL_SHA256", + "TLS_ECDHE_ECDSA_WITH_NULL_SHA" + ); + + /** */ + @Rule + public final ExpectedException ruleForExpectedException = ExpectedException.none(); + + /** + * @param jettyCfg Optional path to file with Jetty XML config. + * @return Prepare configuration for cluster node. + */ + private IgniteConfiguration nodeConfiguration(String jettyCfg) { + TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(); + + ipFinder.registerAddresses(Collections.singletonList(new InetSocketAddress("127.0.0.1", 47500))); + + TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi(); + + discoverySpi.setIpFinder(ipFinder); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setDiscoverySpi(discoverySpi); + + CacheConfiguration dfltCacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + + cfg.setCacheConfiguration(dfltCacheCfg); + + cfg.setIgniteInstanceName(UUID.randomUUID().toString()); + + if (!F.isEmpty(jettyCfg)) { + ConnectorConfiguration conCfg = new ConnectorConfiguration(); + conCfg.setJettyPath(resolvePath(jettyCfg)); + + cfg.setConnectorConfiguration(conCfg); + } + + return cfg; + } + + /** + * Convert response to JSON. + * + * @param res REST result. + * @return JSON object. + * @throws IOException If failed to parse. + */ + private JsonNode toJson(RestResult res) throws IOException { + Assert.assertNotNull(res); + + String data = res.getData(); + + Assert.assertNotNull(data); + Assert.assertFalse(data.isEmpty()); + + return MAPPER.readTree(data); + } + + /** + * @param file File name. + * @return Path to file. + */ + private String resolvePath(String file) { + return IgniteUtils.resolveIgnitePath(PATH_TO_RESOURCES + file).getAbsolutePath(); + } + + /** + * Try to execute REST command and check response. + * + * @param nodeCfg Node configuration. + * @param uri Node URI. + * @param keyStore Key store. + * @param keyStorePwd Key store password. + * @param trustStore Trust store. + * @param trustStorePwd Trust store password. + * @param cipherSuites Cipher suites. + * @throws Exception If failed. + */ + private void checkRest( + IgniteConfiguration nodeCfg, + String uri, + String keyStore, + String keyStorePwd, + String trustStore, + String trustStorePwd, + List cipherSuites + ) throws Exception { + try( + Ignite ignite = Ignition.getOrStart(nodeCfg); + RestExecutor exec = new RestExecutor(keyStore, keyStorePwd, trustStore, trustStorePwd, cipherSuites) + ) { + Map params = new HashMap<>(); + params.put("cmd", "top"); + params.put("attr", false); + params.put("mtr", false); + params.put("caches", false); + + RestResult res = exec.sendRequest(Collections.singletonList(uri), params, null); + + JsonNode json = toJson(res); + + Assert.assertTrue(json.isArray()); + + for (JsonNode item : json) { + Assert.assertTrue(item.get("attributes").isNull()); + Assert.assertTrue(item.get("metrics").isNull()); + Assert.assertTrue(item.get("caches").isNull()); + } + } + } + + /** */ + @Test + public void nodeNoSslAgentNoSsl() throws Exception { + checkRest( + nodeConfiguration(""), + HTTP_URI, + null, null, + null, null, + null + ); + } + + /** */ + @Test + public void nodeNoSslAgentWithSsl() throws Exception { + // Check Web Agent with SSL. + ruleForExpectedException.expect(SSLHandshakeException.class); + checkRest( + nodeConfiguration(""), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + null + ); + } + + /** */ + @Test + public void nodeWithSslAgentNoSsl() throws Exception { + ruleForExpectedException.expect(IOException.class); + checkRest( + nodeConfiguration(JETTY_WITH_SSL), + HTTP_URI, + null, null, + null, null, + null + ); + } + + /** */ + @Test + public void nodeWithSslAgentWithSsl() throws Exception { + checkRest( + nodeConfiguration(JETTY_WITH_SSL), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + null + ); + } + + /** */ + @Test + public void nodeNoCiphersAgentWithCiphers() throws Exception { + ruleForExpectedException.expect(SSLHandshakeException.class); + checkRest( + nodeConfiguration(JETTY_WITH_SSL), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + CIPHER_0 + ); + } + + /** */ + @Test + public void nodeWithCiphersAgentNoCiphers() throws Exception { + ruleForExpectedException.expect(SSLHandshakeException.class); + checkRest( + nodeConfiguration(JETTY_WITH_CIPHERS_0), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + null + ); + } + + /** */ + @Test + public void nodeWithCiphersAgentWithCiphers() throws Exception { + checkRest( + nodeConfiguration(JETTY_WITH_CIPHERS_1), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + CIPHER_1 + ); + } + + /** */ + @Test + public void differentCiphers1() throws Exception { + ruleForExpectedException.expect(SSLHandshakeException.class); + checkRest( + nodeConfiguration(JETTY_WITH_CIPHERS_1), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + CIPHER_2 + ); + } + + /** */ + @Test + public void differentCiphers2() throws Exception { + ruleForExpectedException.expect(SSLHandshakeException.class); + checkRest( + nodeConfiguration(JETTY_WITH_CIPHERS_2), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + CIPHER_1 + ); + } + + /** */ + @Test + public void commonCiphers() throws Exception { + checkRest( + nodeConfiguration(JETTY_WITH_CIPHERS_1), + HTTPS_URI, + resolvePath("client.jks"), "123456", + resolvePath("ca.jks"), "123456", + COMMON_CIPHERS + ); + } +} diff --git a/modules/web-console/web-agent/src/test/java/org/apache/ignite/testsuites/IgniteWebAgentTestSuite.java b/modules/web-console/web-agent/src/test/java/org/apache/ignite/testsuites/IgniteWebAgentTestSuite.java new file mode 100644 index 0000000000000..d0bc23841a8d0 --- /dev/null +++ b/modules/web-console/web-agent/src/test/java/org/apache/ignite/testsuites/IgniteWebAgentTestSuite.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import org.apache.ignite.console.agent.rest.RestExecutorSelfTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Web Agent tests. + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + RestExecutorSelfTest.class +}) +public class IgniteWebAgentTestSuite { + // No-op. +} diff --git a/modules/web-console/web-agent/src/test/resources/ca.jks b/modules/web-console/web-agent/src/test/resources/ca.jks new file mode 100644 index 0000000000000000000000000000000000000000..9d50bcb2d9618ef4f2642f531f8a750dfecb325b GIT binary patch literal 1394 zcmezO_TO6u1_mY|W&~qKLm(^NLH1x5P;{q36Vp}$J~l3GHbxdkEha%mMpg!vCZ>fk z%>UTfjP8f+{@PpjzSB5Eb=RI8PWRJiIGA6pOnPB*yo6{SbJwT zhsku6J<|J9?_K9Hl2;CW+c!lu>ss7YxjEnNNt?CFmg_b@c6VxC96s^p|3<+L6Rz%& zaCBe)rf^;8%*N--4eGB>n}2WKtSiUY>AlqZ@wsu~=X_pI)F zM}U%cjNu+dt4SQs6OU<5E1mD`*KqwDkITv|{v#}EA(f%x(@uGXzq-WvO>W(b{WIbx z+>BV8rM=^{*3UZr7Y>5KR_rBCEIVRKBCiBQtaq8^Y@u^ZkR1uAEa#vhIEN zb*KIYB?(D=Ra=9A8~65q?ld@atTSY0bN$(rJ{I8>v*(>-p5}LI!tPafGn_QdUi><$ zcPHE7&mp!cGlL?%R6nU&wt&&uv@5mb5}~i z%%%EG)#D$ob-5~t5?XqteV5={BasfcOJSNt>$#_%bDA1ny^Hw)X2~7zTH2OsWHvY^Nu z#L3WoWV-a-X+fz5ylk9WZ60mkc^MhGSs4uK47m+B*_cCF*o2vaLJfHhxIr8a9+r^Q z;u08_9m?eg>ITYk@vu1-BqnF18VVT*fCQL%ICJuo6LT{1fqJ0kaq}>Fh8W5l$inU8 zWE2x9E-fg?FDh{;0h+FtoS$nTC(dhVVQ6S%Y-nI)X!-Nf>ssW#SKXmo0xdroF1eq#w(oZmce(T@$7!X? z30XR!ISKp4eE7TW=(62W=~%nqZo=Y6{+sk2UBadNvirVH$lMvq#LURRi0nLIgfau& z#V|c%ZkN-hPgS!PNc$g{eV=_3SEFr^Vf}`fn*JTe-*PnjyB=os?z!|PZPBx|!=d{o zvBsw)NIh_#r@^c^Q}$ykhn4zCMVn32<5f5DeY$(Wrp@}+oDF`)f((xugSKq&S;f9G zqA+yl`!+rsw&r<34>oW&7i>6Lsi9ce(ET#Gc`LiK>%t}HZU`)A5-n~KQoT0SkzZH> E00d?%U;qFB literal 0 HcmV?d00001 diff --git a/modules/web-console/web-agent/src/test/resources/client.jks b/modules/web-console/web-agent/src/test/resources/client.jks new file mode 100644 index 0000000000000000000000000000000000000000..197c75b6d0acfec70e5ad40841f80e184a4149e5 GIT binary patch literal 2030 zcmezO_TO6u1_mY|W&~qKLm(^NL3U>gP;`$$6Vna@J~l3GHbxdkEha%mMpg!vCZ;7_ zJ5mi!_nx-QwUN0nM__s4#sKBz@xT3MuIX83Dv}To`bW>?zh3=R3zNOZ*DSIu4Sz3P z*}-U3)jVZSMD;dx3kH`X{=dVNxaQX{`{}jmHd}7KSdv87m+l#%Q_N2~xvDe%@b=VJ ze6VI=!iLR}|CO#*)!QL^7FlRqSRrX3vbcn| zv*xqO9s^6Es(psXwHmInTzS25%?k_8dusC50^fMfTwn4))a%*zxt|}o)%3+3xf#Gv zHt)T&H3P%gFR0+ainzAt~J^40&AfLK*^WNT*()LDU;?#FaKURBrbfvL9 zije=${KV^K%0XtU{MSdSst&Ai+bA$~@6?&wgE?g--c&wV!`QL==aI+r^GYvIdMdzE zb=UCx(y4JjN<2Efj!vt&{rSMdV<%p2PD)Oej(IP$Y0{4>n^|+VUyXj$&$#aGU7tH# z9*m}IMOZr;q>5(goS5Kz_3h4y#quXqHMq8FT1fn!9Z`2Oc1pCD{B_6c&wgGG@iEz! zT=?Wwq!_p1j(G)9J63()@q3AyoUl#Q+^!i*89p8t+wI!H^(Zq$GV`0>=Zrn?s>*kk zbg)R>cp~Ss>*%9N)r;qc@lV^*^sk?5Ma$Wn@@w@LZ)t1{pYHae@6+*_`Zeo4H=Tc2 zk+=9vjo3c-`0w?-Yjh@R{rjWaT08mu`b870zBspTdwr(vdD>~6&EH*GxktZ$JKttzwrrcw>ah7y^&m~3r^Oo5qhQumcXQ(2TaOY z22G5YnHZT2c-c6$+C196^D;7WvoaXe8FCwNvN4CUun98-g&Oi2aDzA;JS-up#U(H< zJCw^0)D4v7;$d?vNKDR1H54)s00}VjaOUJEC+1}21NA`7& znVN^>yvF&+0R}7qm>YW;3>rI`8XFnz-fs@+zP_RCs^H|vud7e2F!H||%zOPDhtu4g ziLz6aGcKs+K4Vjw{NcaS>F2CF`%mA_NI9lk$LX`@=nLh=)^#G&66c(1Q~0&&cGr{C zmb6z%?XZnr_~l*{b5C!E z(Ch5fSdX+L@5C&xTr#@dyp*H-+rp;lY4Ixq7EBVq@?G`7f>3$;n?IK+gkNxL+_1Ls z&|Tp~*ZG&3zuF$1vT}Xx)6;)%Wj=DR5WiNcS9k1spRQ}xnLw@VKEAEnMR>aO?DI?3y&xZLYn z zu7_E@doI06Tl6gLaOl2Ctnn!cQV*QxX)r6!l>OMsVWoaj(Pq>1c-2jOpYC3;X|ujH zXM>-yAj6}^pe-AGR "server.cnf" + +:: Generate client config. +( +echo [req] +echo prompt = no +echo distinguished_name = dn +echo req_extensions = req_ext + +echo [ dn ] +echo countryName = RU +echo stateOrProvinceName = Test +echo localityName = Test +echo organizationName = Apache +echo commonName = %CLIENT_DOMAIN_NAME% +echo organizationalUnitName = IT +echo emailAddress = %CLIENT_EMAIL% + +echo [ req_ext ] +echo subjectAltName = @alt_names + +echo [ alt_names ] +echo DNS.1 = %CLIENT_DOMAIN_NAME% +) > "client.cnf" + +:: Generate certificates. +openssl genrsa -des3 -passout pass:%PWD% -out server.key 1024 +openssl req -new -passin pass:%PWD% -key server.key -config server.cnf -out server.csr + +openssl req -new -newkey rsa:1024 -nodes -keyout ca.key -x509 -days 365 -config server.cnf -out ca.crt + +openssl x509 -req -days 365 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -extensions req_ext -extfile server.cnf -out server.crt +openssl rsa -passin pass:%PWD% -in server.key -out server.nopass.key + +openssl req -new -utf8 -nameopt multiline,utf8 -newkey rsa:1024 -nodes -keyout client.key -config client.cnf -out client.csr +openssl x509 -req -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 02 -out client.crt + +openssl pkcs12 -export -in server.crt -inkey server.key -certfile server.crt -out server.p12 -passin pass:%PWD% -passout pass:%PWD% +openssl pkcs12 -export -in client.crt -inkey client.key -certfile ca.crt -out client.p12 -passout pass:%PWD% +openssl pkcs12 -export -in ca.crt -inkey ca.key -certfile ca.crt -out ca.p12 -passout pass:%PWD% + +keytool -importkeystore -srckeystore server.p12 -srcstoretype PKCS12 -destkeystore server.jks -deststoretype JKS -noprompt -srcstorepass %PWD% -deststorepass %PWD% +keytool -importkeystore -srckeystore client.p12 -srcstoretype PKCS12 -destkeystore client.jks -deststoretype JKS -noprompt -srcstorepass %PWD% -deststorepass %PWD% +keytool -importkeystore -srckeystore ca.p12 -srcstoretype PKCS12 -destkeystore ca.jks -deststoretype JKS -noprompt -srcstorepass %PWD% -deststorepass %PWD% + +openssl x509 -text -noout -in server.crt +openssl x509 -text -noout -in client.crt +openssl x509 -text -noout -in ca.crt diff --git a/modules/web-console/web-agent/src/test/resources/generate.sh b/modules/web-console/web-agent/src/test/resources/generate.sh new file mode 100644 index 0000000000000..95e62c395702a --- /dev/null +++ b/modules/web-console/web-agent/src/test/resources/generate.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# SSL certificates generation. +# + +# +# Preconditions: +# 1. If needed, install Open SSL (for example: "sudo apt-get install openssl") +# 2. If needed, install JDK 8 or newer. We need "keytool" from "JDK/bin". +# 3. Create "openssl.cnf" in some folder (for example: "/opt/openssl"). +# You may use "https://github.com/openssl/openssl/blob/master/apps/openssl.cnf" as template. +# 4. If needed, add "opensll" & "keytool" to PATH variable. +# +# NOTE: In case of custom SERVER_DOMAIN_NAME you may need to tweak your "etc/hosts" file. +# + +set -x + +# Set Open SSL variables. +OPENSSL_CONF=/opt/openssl/openssl.cnf + +# Certificates password. +PWD=p123456 + +# Server. +SERVER_DOMAIN_NAME=localhost +SERVER_EMAIL=support@test.local + +# Client. +CLIENT_DOMAIN_NAME=localhost +CLIENT_EMAIL=client@test.local + +# Cleanup. +rm -vf server.* +rm -vf client.* +rm -vf ca.* + +# Generate server config. +cat << EOF > server.cnf +[req] +prompt = no +distinguished_name = dn +req_extensions = req_ext +[ dn ] +countryName = RU +stateOrProvinceName = Moscow +localityName = Moscow +organizationName = test +commonName = ${SERVER_DOMAIN_NAME} +organizationalUnitName = IT +emailAddress = ${SERVER_EMAIL} +[ req_ext ] +subjectAltName = @alt_names +[ alt_names ] +DNS.1 = ${SERVER_DOMAIN_NAME} +EOF + +# Generate client config. +cat << EOF > client.cnf +[req] +prompt = no +distinguished_name = dn +req_extensions = req_ext +[ dn ] +countryName = RU +stateOrProvinceName = Moscow +localityName = Moscow +organizationName = test +commonName = ${CLIENT_DOMAIN_NAME} +organizationalUnitName = IT +emailAddress = ${CLIENT_EMAIL} +[ req_ext ] +subjectAltName = @alt_names +[ alt_names ] +DNS.1 = ${CLIENT_DOMAIN_NAME} +EOF + +# Generate certificates. +openssl genrsa -des3 -passout pass:${PWD} -out server.key 1024 +openssl req -new -passin pass:${PWD} -key server.key -config server.cnf -out server.csr +openssl req -new -newkey rsa:1024 -nodes -keyout ca.key -x509 -days 365 -config server.cnf -out ca.crt +openssl x509 -req -days 365 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -extensions req_ext -extfile server.cnf -out server.crt +openssl rsa -passin pass:${PWD} -in server.key -out server.nopass.key +openssl req -new -utf8 -nameopt multiline,utf8 -newkey rsa:1024 -nodes -keyout client.key -config client.cnf -out client.csr +openssl x509 -req -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 02 -out client.crt +openssl pkcs12 -export -in server.crt -inkey server.key -certfile server.crt -out server.p12 -passin pass:${PWD} -passout pass:${PWD} +openssl pkcs12 -export -in client.crt -inkey client.key -certfile ca.crt -out client.p12 -passout pass:${PWD} +openssl pkcs12 -export -in ca.crt -inkey ca.key -certfile ca.crt -out ca.p12 -passout pass:${PWD} +keytool -importkeystore -srckeystore server.p12 -srcstoretype PKCS12 -destkeystore server.jks -deststoretype JKS -noprompt -srcstorepass ${PWD} -deststorepass ${PWD} +keytool -importkeystore -srckeystore client.p12 -srcstoretype PKCS12 -destkeystore client.jks -deststoretype JKS -noprompt -srcstorepass ${PWD} -deststorepass ${PWD} +keytool -importkeystore -srckeystore ca.p12 -srcstoretype PKCS12 -destkeystore ca.jks -deststoretype JKS -noprompt -srcstorepass ${PWD} -deststorepass ${PWD} +openssl x509 -text -noout -in server.crt +openssl x509 -text -noout -in client.crt +openssl x509 -text -noout -in ca.crt diff --git a/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-0.xml b/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-0.xml new file mode 100644 index 0000000000000..40f08b552d97d --- /dev/null +++ b/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-0.xml @@ -0,0 +1,94 @@ + + + + + + + + + 5 + 10 + + + + + https + + true + true + + + + + + + modules/web-console/web-agent/src/test/resources/server.jks + 123456 + modules/web-console/web-agent/src/test/resources/ca.jks + 123456 + true + + + TLS_DH_anon_WITH_AES_256_GCM_SHA384 + + + + + + + + + + + + + + + + http/1.1 + + + + + + + + + + + + 30000 + true + + + + + + + + + + + + + + + + + false + diff --git a/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-1.xml b/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-1.xml new file mode 100644 index 0000000000000..cb3a293d5b796 --- /dev/null +++ b/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-1.xml @@ -0,0 +1,94 @@ + + + + + + + + + 5 + 10 + + + + + https + + true + true + + + + + + + modules/web-console/web-agent/src/test/resources/server.jks + 123456 + modules/web-console/web-agent/src/test/resources/ca.jks + 123456 + true + + + TLS_RSA_WITH_NULL_SHA256 + + + + + + + + + + + + + + + + http/1.1 + + + + + + + + + + + + 30000 + true + + + + + + + + + + + + + + + + + false + diff --git a/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-2.xml b/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-2.xml new file mode 100644 index 0000000000000..2251de26716ea --- /dev/null +++ b/modules/web-console/web-agent/src/test/resources/jetty-with-ciphers-2.xml @@ -0,0 +1,94 @@ + + + + + + + + + 5 + 10 + + + + + https + + true + true + + + + + + + modules/web-console/web-agent/src/test/resources/server.jks + 123456 + modules/web-console/web-agent/src/test/resources/ca.jks + 123456 + true + + + TLS_ECDHE_ECDSA_WITH_NULL_SHA + + + + + + + + + + + + + + + + http/1.1 + + + + + + + + + + + + 30000 + true + + + + + + + + + + + + + + + + + false + diff --git a/modules/web-console/web-agent/src/test/resources/jetty-with-ssl.xml b/modules/web-console/web-agent/src/test/resources/jetty-with-ssl.xml new file mode 100644 index 0000000000000..7e0682943a66c --- /dev/null +++ b/modules/web-console/web-agent/src/test/resources/jetty-with-ssl.xml @@ -0,0 +1,89 @@ + + + + + + + + + 5 + 10 + + + + + https + + true + true + + + + + + + modules/web-console/web-agent/src/test/resources/server.jks + 123456 + modules/web-console/web-agent/src/test/resources/ca.jks + 123456 + true + + + + + + + + + + + + + + http/1.1 + + + + + + + + + + + + 30000 + true + + + + + + + + + + + + + + + + + false + diff --git a/modules/web-console/web-agent/src/test/resources/server.jks b/modules/web-console/web-agent/src/test/resources/server.jks new file mode 100644 index 0000000000000000000000000000000000000000..c673bb0c8d6574e8b509179c9b97c3c943da1f2d GIT binary patch literal 1419 zcmezO_TO6u1_mY|W&~qKLm(^NL3ZP0py+OcCZ_EMd~96WY>X_7T1Fe``#fXT@-1*MB>gwRVbIIj;0%1SmFK^)}h^NcIfp+~0rZZp*w}D8jn8@k)Vp zbj|t_wc={0n8?R%yG1x%_k|v+4-Wfxn8!fRb&dX;iYcG1{wT%F{K4-0ko}C+$aZ41-|9FupP+7o+Mr+jxq6AA zx6`M(>4Y<%EwHJ1Yw~6V=YvMum+PrfULa(%^&D7cwC{vVO9{5Ie zgUIZU{`(EX_AmVF;hyd4Z{l$!oZJX>(i(yJmxZ;r-?RGtgZB(MEz z+-fy>{p^C3HT^uIEBdBy-t&5jd}U9#saI9vdv)K*aSwybeRFTjjQP7Md9pJ{LO`*R0P&ZJPi-*mzATc>3)lkSl03^W7!kztW=*Sk#tcHLF)BG)Ft#aw6=W`VjzGz9lygW5Cc>1dM zEUF64&g|Y7&lw14lw5L|r~R(4W#XI zFSPgeoF)2C8#X={NYm7 Date: Wed, 26 Dec 2018 18:44:59 +0300 Subject: [PATCH 290/403] IGNITE-10779 Call writtenPagesCntr incrementor - Fixes #5721. Signed-off-by: Dmitriy Pavlov (cherry picked from commit 7ff040438319b55fd81c467bacb644efa3afaf92) --- .../cache/persistence/GridCacheDatabaseSharedManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index d6c96b0f57d65..6199896389c9a 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -4080,6 +4080,8 @@ else if (grpId == TxLog.TX_LOG_CACHE_ID) tracker.onDataPageWritten(); } + writtenPagesCntr.incrementAndGet(); + PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, true); updStores.computeIfAbsent(store, k -> new LongAdder()).increment(); From b6bd81a8a82676e566ae99aa2ba1ddb2ccef2091 Mon Sep 17 00:00:00 2001 From: Ilya Borisov Date: Wed, 19 Dec 2018 10:03:31 +0700 Subject: [PATCH 291/403] IGNITE-9839 Web Console: update to RxJS 6. (cherry picked from commit 7d4e1fd118845f6e14638520ac10881deadd570c) --- .../components/cluster-selector/controller.js | 17 +- .../connected-clusters-badge/controller.js | 11 +- .../components/cache-edit-form/controller.js | 10 +- .../cluster-edit-form/controller.js | 10 +- .../controller.js | 36 +- .../controller.js | 29 +- .../controller.js | 49 +- .../controller.js | 45 +- .../page-configure-basic/controller.js | 87 ++- .../page-configure-basic/controller.spec.js | 6 +- .../page-configure-overview/controller.js | 11 +- .../modal-import-models/component.js | 111 +-- .../components/page-configure/controller.js | 15 +- .../app/components/page-configure/index.d.ts | 29 +- .../app/components/page-configure/index.ts | 26 +- .../services/ConfigChangesGuard.js | 11 +- .../services/ConfigSelectionManager.js | 75 +- .../page-configure/services/ConfigureState.js | 11 +- .../page-configure/services/PageConfigure.js | 49 +- .../services/PageConfigure.spec.js | 17 +- .../app/components/page-configure/states.js | 62 +- .../page-configure/store/effects.js | 638 ++++++++++-------- .../page-configure/store/effects.spec.js | 7 +- .../page-configure/store/selectors.js | 159 ++--- .../components/queries-notebook/controller.ts | 30 +- .../app/modules/agent/AgentManager.service.js | 25 +- .../frontend/app/modules/user/User.service.js | 2 +- .../frontend/app/services/Clusters.js | 9 +- .../frontend/app/services/Version.service.js | 2 +- .../frontend/app/services/store.ts | 10 +- .../web-console/frontend/app/store/ofType.ts | 2 +- .../frontend/app/utils/SimpleWorkerPool.js | 19 +- modules/web-console/frontend/package.json | 4 +- 33 files changed, 869 insertions(+), 755 deletions(-) diff --git a/modules/web-console/frontend/app/components/cluster-selector/controller.js b/modules/web-console/frontend/app/components/cluster-selector/controller.js index fb64b9f229eed..484a5eb4bc893 100644 --- a/modules/web-console/frontend/app/components/cluster-selector/controller.js +++ b/modules/web-console/frontend/app/components/cluster-selector/controller.js @@ -17,8 +17,8 @@ import _ from 'lodash'; -import { BehaviorSubject } from 'rxjs/BehaviorSubject'; -import 'rxjs/add/operator/combineLatest'; +import { BehaviorSubject } from 'rxjs'; +import {tap, filter, combineLatest} from 'rxjs/operators'; export default class { static $inject = ['AgentManager', 'IgniteConfirm', 'IgniteVersion', 'IgniteMessages']; @@ -46,15 +46,16 @@ export default class { this.inProgress$ = this._inProgressSubject.asObservable(); - this.clusters$ = this.agentMgr.connectionSbj - .combineLatest(this.inProgress$) - .do(([sbj, inProgress]) => this.inProgress = inProgress) - .filter(([sbj, inProgress]) => !inProgress) - .do(([{cluster, clusters}]) => { + this.clusters$ = this.agentMgr.connectionSbj.pipe( + combineLatest(this.inProgress$), + tap(([sbj, inProgress]) => this.inProgress = inProgress), + filter(([sbj, inProgress]) => !inProgress), + tap(([{cluster, clusters}]) => { this.cluster = cluster ? {...cluster} : null; this.clusters = _.orderBy(clusters, ['name'], ['asc']); }) - .subscribe(() => {}); + ) + .subscribe(() => {}); } $onDestroy() { diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js index 60a06965e2a65..896c02bd90b02 100644 --- a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js +++ b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js @@ -15,6 +15,8 @@ * limitations under the License. */ +import {tap} from 'rxjs/operators'; + export default class { static $inject = ['$scope', 'AgentManager', 'ConnectedClustersDialog']; @@ -38,13 +40,14 @@ export default class { } $onInit() { - this.connectedClusters$ = this.agentMgr.connectionSbj - .do(({ clusters }) => this.connectedClusters = clusters.length) - .do(({ clusters }) => { + this.connectedClusters$ = this.agentMgr.connectionSbj.pipe( + tap(({ clusters }) => this.connectedClusters = clusters.length), + tap(({ clusters }) => { this.clusters = clusters; this.$scope.$applyAsync(); }) - .subscribe(); + ) + .subscribe(); } $onDestroy() { diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js index e044a7012d589..c1beec648a82f 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js @@ -17,6 +17,7 @@ import cloneDeep from 'lodash/cloneDeep'; import get from 'lodash/get'; +import {tap} from 'rxjs/operators'; export default class CacheEditFormController { /** @type {ig.menu} */ @@ -61,10 +62,11 @@ export default class CacheEditFormController { }; - this.subscription = this.IgniteVersion.currentSbj - .do(rebuildDropdowns) - .do(filterModel) - .subscribe(); + this.subscription = this.IgniteVersion.currentSbj.pipe( + tap(rebuildDropdowns), + tap(filterModel) + ) + .subscribe(); // TODO: Do we really need this? this.$scope.ui = this.IgniteFormUtils.formUI(); diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js index 5020d7bd18a56..7400b4374b851 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js @@ -19,6 +19,7 @@ import cloneDeep from 'lodash/cloneDeep'; import get from 'lodash/get'; import isEqual from 'lodash/isEqual'; import _ from 'lodash'; +import {tap} from 'rxjs/operators'; export default class ClusterEditFormController { /** @type {Array} */ @@ -86,10 +87,11 @@ export default class ClusterEditFormController { } }; - this.subscription = this.IgniteVersion.currentSbj - .do(rebuildDropdowns) - .do(() => filterModel(this.clonedCluster)) - .subscribe(); + this.subscription = this.IgniteVersion.currentSbj.pipe( + tap(rebuildDropdowns), + tap(() => filterModel(this.clonedCluster)) + ) + .subscribe(); this.supportedJdbcTypes = this.IgniteLegacyUtils.mkOptions(this.IgniteLegacyUtils.SUPPORTED_JDBC_TYPES); diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js index da17c678fe6c2..2cfe08ae80d70 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js @@ -15,10 +15,9 @@ * limitations under the License. */ -import {Subject} from 'rxjs/Subject'; -import {merge} from 'rxjs/observable/merge'; +import {Subject, merge, combineLatest} from 'rxjs'; +import {tap, map, refCount, pluck, publishReplay, switchMap, distinctUntilChanged} from 'rxjs/operators'; import naturalCompare from 'natural-compare-lite'; -import {combineLatest} from 'rxjs/observable/combineLatest'; import {removeClusterItems, advancedSaveCache} from 'app/components/page-configure/store/actionCreators'; import ConfigureState from 'app/components/page-configure/services/ConfigureState'; import ConfigSelectors from 'app/components/page-configure/store/selectors'; @@ -103,16 +102,23 @@ export default class Controller { } $onInit() { - const cacheID$ = this.$uiRouter.globals.params$.pluck('cacheID').publishReplay(1).refCount(); + const cacheID$ = this.$uiRouter.globals.params$.pipe( + pluck('cacheID'), + publishReplay(1), + refCount() + ); - this.shortCaches$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortCaches); - this.shortModels$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortModels); - this.shortIGFSs$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortIGFSs); - this.originalCache$ = cacheID$.distinctUntilChanged().switchMap((id) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectCacheToEdit(id)); - }); + this.shortCaches$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCurrentShortCaches); + this.shortModels$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCurrentShortModels); + this.shortIGFSs$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCurrentShortIGFSs); + this.originalCache$ = cacheID$.pipe( + distinctUntilChanged(), + switchMap((id) => { + return this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCacheToEdit(id)); + }) + ); - this.isNew$ = cacheID$.map((id) => id === 'new'); + this.isNew$ = cacheID$.pipe(map((id) => id === 'new')); this.itemEditTitle$ = combineLatest(this.isNew$, this.originalCache$, (isNew, cache) => { return `${isNew ? 'Create' : 'Edit'} cache ${!isNew && cache.name ? `‘${cache.name}’` : ''}`; }); @@ -125,13 +131,13 @@ export default class Controller { this.subscription = merge( this.originalCache$, - this.selectionManager.editGoes$.do((id) => this.edit(id)), - this.selectionManager.editLeaves$.do((options) => this.$state.go('base.configuration.edit.advanced.caches', null, options)) + this.selectionManager.editGoes$.pipe(tap((id) => this.edit(id))), + this.selectionManager.editLeaves$.pipe(tap((options) => this.$state.go('base.configuration.edit.advanced.caches', null, options))) ).subscribe(); this.isBlocked$ = cacheID$; - this.tableActions$ = this.selectionManager.selectedItemIDs$.map((selectedItems) => [ + this.tableActions$ = this.selectionManager.selectedItemIDs$.pipe(map((selectedItems) => [ { action: 'Clone', click: () => this.clone(selectedItems), @@ -144,7 +150,7 @@ export default class Controller { }, available: true } - ]); + ])); } /** diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js index f0348c353629e..2aae041ce8349 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js @@ -18,7 +18,7 @@ import {default as ConfigSelectors} from 'app/components/page-configure/store/selectors'; import {default as ConfigureState} from 'app/components/page-configure/services/ConfigureState'; import {advancedSaveCluster} from 'app/components/page-configure/store/actionCreators'; -import 'rxjs/add/operator/publishReplay'; +import {take, pluck, switchMap, map, filter, distinctUntilChanged, publishReplay, refCount} from 'rxjs/operators'; // Controller for Clusters screen. export default class PageConfigureAdvancedCluster { @@ -36,12 +36,27 @@ export default class PageConfigureAdvancedCluster { } $onInit() { - const clusterID$ = this.$uiRouter.globals.params$.take(1).pluck('clusterID').filter((v) => v).take(1); - this.shortCaches$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortCaches); - this.originalCluster$ = clusterID$.distinctUntilChanged().switchMap((id) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id)); - }).distinctUntilChanged().publishReplay(1).refCount(); - this.isNew$ = this.$uiRouter.globals.params$.pluck('clusterID').map((id) => id === 'new'); + const clusterID$ = this.$uiRouter.globals.params$.pipe( + take(1), + pluck('clusterID'), + filter((v) => v), + take(1) + ); + + this.shortCaches$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCurrentShortCaches); + + this.originalCluster$ = clusterID$.pipe( + distinctUntilChanged(), + switchMap((id) => { + return this.ConfigureState.state$.pipe(this.ConfigSelectors.selectClusterToEdit(id)); + }), + distinctUntilChanged(), + publishReplay(1), + refCount() + ); + + this.isNew$ = this.$uiRouter.globals.params$.pipe(pluck('clusterID'), map((id) => id === 'new')); + this.isBlocked$ = clusterID$; } diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js index 50eee7cad0dce..0e6345b088eec 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js @@ -15,11 +15,9 @@ * limitations under the License. */ -import {Observable} from 'rxjs/Observable'; -import {Subject} from 'rxjs/Subject'; -import {combineLatest} from 'rxjs/observable/combineLatest'; +import {Observable, Subject, combineLatest, merge} from 'rxjs'; +import {tap, map, distinctUntilChanged, pluck, publishReplay, refCount, switchMap} from 'rxjs/operators'; import naturalCompare from 'natural-compare-lite'; -import {merge} from 'rxjs/observable/merge'; import get from 'lodash/get'; import {removeClusterItems, advancedSaveIGFS} from 'app/components/page-configure/store/actionCreators'; import ConfigureState from 'app/components/page-configure/services/ConfigureState'; @@ -82,31 +80,44 @@ export default class PageConfigureAdvancedIGFS { width: 130 } ]; - this.itemID$ = this.$uiRouter.globals.params$.pluck('igfsID'); + + this.itemID$ = this.$uiRouter.globals.params$.pipe(pluck('igfsID')); /** @type {Observable} */ - this.shortItems$ = this.ConfigureState.state$ - .let(this.ConfigSelectors.selectCurrentShortIGFSs) - .map((items = []) => items.map((i) => ({ + this.shortItems$ = this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectCurrentShortIGFSs, + map((items = []) => items.map((i) => ({ _id: i._id, name: i.name, affinnityGroupSize: i.affinnityGroupSize || this.IGFSs.affinnityGroupSize.default, defaultMode: i.defaultMode || this.IGFSs.defaultMode.default - }))); - this.originalItem$ = this.itemID$.distinctUntilChanged().switchMap((id) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectIGFSToEdit(id)); - }).distinctUntilChanged().publishReplay(1).refCount(); - this.isNew$ = this.itemID$.map((id) => id === 'new'); + }))) + ); + + this.originalItem$ = this.itemID$.pipe( + distinctUntilChanged(), + switchMap((id) => { + return this.ConfigureState.state$.pipe(this.ConfigSelectors.selectIGFSToEdit(id)); + }), + distinctUntilChanged(), + publishReplay(1), + refCount() + ); + + this.isNew$ = this.itemID$.pipe(map((id) => id === 'new')); + this.itemEditTitle$ = combineLatest(this.isNew$, this.originalItem$, (isNew, item) => { return `${isNew ? 'Create' : 'Edit'} IGFS ${!isNew && get(item, 'name') ? `‘${get(item, 'name')}’` : ''}`; }); + this.selectionManager = this.configSelectionManager({ itemID$: this.itemID$, selectedItemRows$: this.selectedRows$, visibleRows$: this.visibleRows$, loadedItems$: this.shortItems$ }); - this.tableActions$ = this.selectionManager.selectedItemIDs$.map((selectedItems) => [ + + this.tableActions$ = this.selectionManager.selectedItemIDs$.pipe(map((selectedItems) => [ { action: 'Clone', click: () => this.clone(selectedItems), @@ -119,19 +130,23 @@ export default class PageConfigureAdvancedIGFS { }, available: true } - ]); + ])); + this.subscription = merge( this.originalItem$, - this.selectionManager.editGoes$.do((id) => this.edit(id)), - this.selectionManager.editLeaves$.do((options) => this.$state.go('base.configuration.edit.advanced.igfs', null, options)) + this.selectionManager.editGoes$.pipe(tap((id) => this.edit(id))), + this.selectionManager.editLeaves$.pipe(tap((options) => this.$state.go('base.configuration.edit.advanced.igfs', null, options))) ).subscribe(); } + edit(igfsID) { this.$state.go('base.configuration.edit.advanced.igfs.igfs', {igfsID}); } + save({igfs, download}) { this.ConfigureState.dispatchAction(advancedSaveIGFS(igfs, download)); } + remove(itemIDs) { this.ConfigureState.dispatchAction( removeClusterItems(this.$uiRouter.globals.params.clusterID, 'igfss', itemIDs, true, true) diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js index fd1ccaa703a72..c2e30cfdf1960 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js @@ -15,10 +15,9 @@ * limitations under the License. */ -import {Subject} from 'rxjs/Subject'; -import {Observable} from 'rxjs/Observable'; -import {combineLatest} from 'rxjs/observable/combineLatest'; -import {merge} from 'rxjs/observable/merge'; +import {Subject, Observable, combineLatest, merge} from 'rxjs'; +import {pluck, tap, publishReplay, refCount, distinctUntilChanged, switchMap, map} from 'rxjs/operators'; + import get from 'lodash/get'; import hasIndexTemplate from './hasIndex.template.pug'; @@ -100,25 +99,33 @@ export default class PageConfigureAdvancedModels { ]; /** @type {Observable} */ - this.itemID$ = this.$uiRouter.globals.params$.pluck('modelID'); + this.itemID$ = this.$uiRouter.globals.params$.pipe(pluck('modelID')); /** @type {Observable>} */ - this.shortItems$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortModels) - .do((shortModels = []) => { + this.shortItems$ = this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectCurrentShortModels, + tap((shortModels = []) => { const value = shortModels.every((m) => m.hasIndex); this.columnDefs[0].visible = !value; - }) - .publishReplay(1) - .refCount(); + }), + publishReplay(1), + refCount() + ); - this.shortCaches$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortCaches); + this.shortCaches$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCurrentShortCaches); /** @type {Observable} */ - this.originalItem$ = this.itemID$.distinctUntilChanged().switchMap((id) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectModelToEdit(id)); - }).distinctUntilChanged().publishReplay(1).refCount(); + this.originalItem$ = this.itemID$.pipe( + distinctUntilChanged(), + switchMap((id) => { + return this.ConfigureState.state$.pipe(this.ConfigSelectors.selectModelToEdit(id)); + }), + distinctUntilChanged(), + publishReplay(1), + refCount() + ); - this.isNew$ = this.itemID$.map((id) => id === 'new'); + this.isNew$ = this.itemID$.pipe(map((id) => id === 'new')); this.itemEditTitle$ = combineLatest(this.isNew$, this.originalItem$, (isNew, item) => { return `${isNew ? 'Create' : 'Edit'} model ${!isNew && get(item, 'valueType') ? `‘${get(item, 'valueType')}’` : ''}`; @@ -131,7 +138,7 @@ export default class PageConfigureAdvancedModels { loadedItems$: this.shortItems$ }); - this.tableActions$ = this.selectionManager.selectedItemIDs$.map((selectedItems) => [ + this.tableActions$ = this.selectionManager.selectedItemIDs$.pipe(map((selectedItems) => [ { action: 'Clone', click: () => this.clone(selectedItems), @@ -144,12 +151,12 @@ export default class PageConfigureAdvancedModels { }, available: true } - ]); + ])); this.subscription = merge( this.originalItem$, - this.selectionManager.editGoes$.do((id) => this.edit(id)), - this.selectionManager.editLeaves$.do((options) => this.$state.go('base.configuration.edit.advanced.models', null, options)) + this.selectionManager.editGoes$.pipe(tap((id) => this.edit(id))), + this.selectionManager.editLeaves$.pipe(tap((options) => this.$state.go('base.configuration.edit.advanced.models', null, options))) ).subscribe(); } diff --git a/modules/web-console/frontend/app/components/page-configure-basic/controller.js b/modules/web-console/frontend/app/components/page-configure-basic/controller.js index f5515fb99d2f3..14b3a69d28aab 100644 --- a/modules/web-console/frontend/app/components/page-configure-basic/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-basic/controller.js @@ -15,8 +15,8 @@ * limitations under the License. */ -import {Observable} from 'rxjs/Observable'; -import 'rxjs/add/operator/map'; +import {forkJoin, merge} from 'rxjs'; +import {map, tap, pluck, take, filter, distinctUntilChanged, switchMap, publishReplay, refCount} from 'rxjs/operators'; import cloneDeep from 'lodash/cloneDeep'; import get from 'lodash/get'; import naturalCompare from 'natural-compare-lite'; @@ -88,45 +88,64 @@ export default class PageConfigureBasicController { $transition$.onSuccess({}, () => this.reset()); - return Observable.forkJoin( - this.ConfigureState.state$.pluck('edit', 'changes').take(1), - this.clusterID$.switchMap((id) => this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterShortCaches(id))).take(1), - this.shortCaches$.take(1) - ).toPromise() - .then(([changes, originalShortCaches, currentCaches]) => { - return this.ConfigChangesGuard.guard( - { - cluster: this.Clusters.normalize(this.originalCluster), - caches: originalShortCaches.map(this.Caches.normalize) - }, - { - cluster: {...this.Clusters.normalize(this.clonedCluster), caches: changes.caches.ids}, - caches: currentCaches.map(this.Caches.normalize) - } - ); - }); + return forkJoin( + this.ConfigureState.state$.pipe(pluck('edit', 'changes'), take(1)), + this.clusterID$.pipe( + switchMap((id) => this.ConfigureState.state$.pipe(this.ConfigSelectors.selectClusterShortCaches(id))), + take(1) + ), + this.shortCaches$.pipe(take(1)) + ) + .toPromise() + .then(([changes, originalShortCaches, currentCaches]) => { + return this.ConfigChangesGuard.guard( + { + cluster: this.Clusters.normalize(this.originalCluster), + caches: originalShortCaches.map(this.Caches.normalize) + }, + { + cluster: {...this.Clusters.normalize(this.clonedCluster), caches: changes.caches.ids}, + caches: currentCaches.map(this.Caches.normalize) + } + ); + }); } $onInit() { this.onBeforeTransition = this.$uiRouter.transitionService.onBefore({}, (t) => this._uiCanExit(t)); - this.memorySizeInputVisible$ = this.IgniteVersion.currentSbj - .map((version) => this.IgniteVersion.since(version.ignite, '2.0.0')); + this.memorySizeInputVisible$ = this.IgniteVersion.currentSbj.pipe( + map((version) => this.IgniteVersion.since(version.ignite, '2.0.0')) + ); - const clusterID$ = this.$uiRouter.globals.params$.take(1).pluck('clusterID').filter((v) => v).take(1); + const clusterID$ = this.$uiRouter.globals.params$.pipe( + take(1), + pluck('clusterID'), + filter((v) => v), + take(1) + ); this.clusterID$ = clusterID$; - this.isNew$ = this.$uiRouter.globals.params$.pluck('clusterID').map((id) => id === 'new'); - this.shortCaches$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectCurrentShortCaches); - this.shortClusters$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectShortClustersValue()); - this.originalCluster$ = clusterID$.distinctUntilChanged().switchMap((id) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id)); - }).distinctUntilChanged().publishReplay(1).refCount(); - - this.subscription = Observable.merge( - this.shortCaches$.map((caches) => caches.sort((a, b) => naturalCompare(a.name, b.name))).do((v) => this.shortCaches = v), - this.shortClusters$.do((v) => this.shortClusters = v), - this.originalCluster$.do((v) => { + this.isNew$ = this.$uiRouter.globals.params$.pipe(pluck('clusterID'), map((id) => id === 'new')); + this.shortCaches$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCurrentShortCaches); + this.shortClusters$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectShortClustersValue()); + this.originalCluster$ = clusterID$.pipe( + distinctUntilChanged(), + switchMap((id) => { + return this.ConfigureState.state$.pipe(this.ConfigSelectors.selectClusterToEdit(id)); + }), + distinctUntilChanged(), + publishReplay(1), + refCount() + ); + + this.subscription = merge( + this.shortCaches$.pipe( + map((caches) => caches.sort((a, b) => naturalCompare(a.name, b.name))), + tap((v) => this.shortCaches = v) + ), + this.shortClusters$.pipe(tap((v) => this.shortClusters = v)), + this.originalCluster$.pipe(tap((v) => { this.originalCluster = v; // clonedCluster should be set only when particular cluster edit starts. // @@ -136,7 +155,7 @@ export default class PageConfigureBasicController { // made by user and we don't want that. Advanced configuration forms do the same too. if (get(v, '_id') !== get(this.clonedCluster, '_id')) this.clonedCluster = cloneDeep(v); this.defaultMemoryPolicy = this.Clusters.getDefaultClusterMemoryPolicy(this.clonedCluster); - }) + })) ).subscribe(); this.formActionsMenu = [ diff --git a/modules/web-console/frontend/app/components/page-configure-basic/controller.spec.js b/modules/web-console/frontend/app/components/page-configure-basic/controller.spec.js index a35eb50a443ee..a4a1d309c4da9 100644 --- a/modules/web-console/frontend/app/components/page-configure-basic/controller.spec.js +++ b/modules/web-console/frontend/app/components/page-configure-basic/controller.spec.js @@ -19,10 +19,8 @@ import {suite, test} from 'mocha'; import {assert} from 'chai'; import {spy} from 'sinon'; -import {TestScheduler} from 'rxjs/testing/TestScheduler'; -import {Subject} from 'rxjs/Subject'; -import {BehaviorSubject} from 'rxjs/BehaviorSubject'; -import {Subscriber} from 'rxjs/Subscriber'; +import {TestScheduler} from 'rxjs/testing'; +import {Subscriber, Subject, BehaviorSubject} from 'rxjs'; import Controller from './controller'; const mocks = () => new Map([ diff --git a/modules/web-console/frontend/app/components/page-configure-overview/controller.js b/modules/web-console/frontend/app/components/page-configure-overview/controller.js index e49e5c68a8bb5..e170112ce76b1 100644 --- a/modules/web-console/frontend/app/components/page-configure-overview/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-overview/controller.js @@ -15,7 +15,8 @@ * limitations under the License. */ -import {Subject} from 'rxjs/Subject'; +import {Subject} from 'rxjs'; +import {map} from 'rxjs/operators'; import naturalCompare from 'natural-compare-lite'; const cellTemplate = (state) => ` @@ -81,7 +82,7 @@ export default class PageConfigureOverviewController { } $onInit() { - this.shortClusters$ = this.ConfigureState.state$.let(this.ConfigSelectors.selectShortClustersValue()); + this.shortClusters$ = this.ConfigureState.state$.pipe(this.ConfigSelectors.selectShortClustersValue()); /** @type {Array>} */ this.clustersColumnDefs = [ @@ -140,9 +141,9 @@ export default class PageConfigureOverviewController { /** @type {Subject>} */ this.selectedRows$ = new Subject(); - this.selectedRowsIDs$ = this.selectedRows$.map((selectedClusters) => selectedClusters.map((cluster) => cluster._id)); + this.selectedRowsIDs$ = this.selectedRows$.pipe(map((selectedClusters) => selectedClusters.map((cluster) => cluster._id))); - this.actions$ = this.selectedRows$.map((selectedClusters) => [ + this.actions$ = this.selectedRows$.pipe(map((selectedClusters) => [ { action: 'Edit', click: () => this.editCluster(selectedClusters[0]), @@ -163,6 +164,6 @@ export default class PageConfigureOverviewController { click: () => this.removeClusters(selectedClusters), available: true } - ]); + ])); } } diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js index e4b82530cdaca..7181b546b3b5c 100644 --- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js +++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js @@ -21,7 +21,8 @@ import _ from 'lodash'; import naturalCompare from 'natural-compare-lite'; import find from 'lodash/fp/find'; import get from 'lodash/fp/get'; -import {Observable} from 'rxjs/Observable'; +import {race, timer, merge, of, from, combineLatest} from 'rxjs'; +import {tap, filter, take, pluck, switchMap} from 'rxjs/operators'; import ObjectID from 'bson-objectid'; import {uniqueName} from 'app/utils/uniqueName'; import {defaultNames} from '../../defaultNames'; @@ -126,34 +127,34 @@ export class ModalImportModels { } loadData() { - return Observable.of(this.clusterID) - .switchMap((id = 'new') => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id, defaultNames.importedCluster)); - }) - .switchMap((cluster) => { - return (!(cluster.caches || []).length && !(cluster.models || []).length) - ? Observable.of({ - cluster, - caches: [], - models: [] - }) - : Observable.fromPromise(Promise.all([ - this.ConfigEffects.etp('LOAD_SHORT_CACHES', {ids: cluster.caches || [], clusterID: cluster._id}), - this.ConfigEffects.etp('LOAD_SHORT_MODELS', {ids: cluster.models || [], clusterID: cluster._id}) - ])) - .switchMap(() => { - return Observable.combineLatest( - this.ConfigureState.state$.let(this.ConfigSelectors.selectShortCachesValue()), - this.ConfigureState.state$.let(this.ConfigSelectors.selectShortModelsValue()), - (caches, models) => ({ - cluster, - caches, - models - }) - ).take(1); - }); - }) - .take(1); + return of(this.clusterID).pipe( + switchMap((id = 'new') => { + return this.ConfigureState.state$.pipe(this.ConfigSelectors.selectClusterToEdit(id, defaultNames.importedCluster)); + }), + switchMap((cluster) => { + return (!(cluster.caches || []).length && !(cluster.models || []).length) + ? of({ + cluster, + caches: [], + models: [] + }) + : from(Promise.all([ + this.ConfigEffects.etp('LOAD_SHORT_CACHES', {ids: cluster.caches || [], clusterID: cluster._id}), + this.ConfigEffects.etp('LOAD_SHORT_MODELS', {ids: cluster.models || [], clusterID: cluster._id}) + ])).pipe(switchMap(() => { + return combineLatest( + this.ConfigureState.state$.pipe(this.ConfigSelectors.selectShortCachesValue()), + this.ConfigureState.state$.pipe(this.ConfigSelectors.selectShortModelsValue()), + (caches, models) => ({ + cluster, + caches, + models + }) + ).pipe(take(1)); + })); + }), + take(1) + ); } saveBatch(batch) { @@ -169,15 +170,20 @@ export class ModalImportModels { prevActions: [] }); - this.saveSubscription = Observable.race( - this.ConfigureState.actions$.filter((a) => a.type === 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK') - .do(() => this.onHide()), - this.ConfigureState.actions$.filter((a) => a.type === 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_ERR') + this.saveSubscription = race( + this.ConfigureState.actions$.pipe( + filter((a) => a.type === 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK'), + tap(() => this.onHide()) + ), + this.ConfigureState.actions$.pipe( + filter((a) => a.type === 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_ERR') + ) + ).pipe( + take(1), + tap(() => { + this.Loading.finish('importDomainFromDb'); + }) ) - .take(1) - .do(() => { - this.Loading.finish('importDomainFromDb'); - }) .subscribe(); } @@ -231,18 +237,23 @@ export class ModalImportModels { if (this.loadedCaches[cacheID]) return; - return this.onCacheSelectSubcription = Observable.merge( - Observable.timer(0, 1).take(1) - .do(() => this.ConfigureState.dispatchAction({type: 'LOAD_CACHE', cacheID})), - Observable.race( - this.ConfigureState.actions$ - .filter((a) => a.type === 'LOAD_CACHE_OK' && a.cache._id === cacheID).pluck('cache') - .do((cache) => { + return this.onCacheSelectSubcription = merge( + timer(0, 1).pipe( + take(1), + tap(() => this.ConfigureState.dispatchAction({type: 'LOAD_CACHE', cacheID})) + ), + race( + this.ConfigureState.actions$.pipe( + filter((a) => a.type === 'LOAD_CACHE_OK' && a.cache._id === cacheID), + pluck('cache'), + tap((cache) => { this.loadedCaches[cacheID] = cache; - }), - this.ConfigureState.actions$ - .filter((a) => a.type === 'LOAD_CACHE_ERR' && a.action.cacheID === cacheID) - ).take(1) + }) + ), + this.ConfigureState.actions$.pipe( + filter((a) => a.type === 'LOAD_CACHE_ERR' && a.action.cacheID === cacheID) + ) + ).pipe(take(1)) ) .subscribe(); } @@ -291,7 +302,7 @@ export class ModalImportModels { this.$scope.importCommon = {}; - this.subscription = this.loadData().do((data) => { + this.subscription = this.loadData().pipe(tap((data) => { this.$scope.caches = _mapCaches(data.caches); this.$scope.domains = data.models; this.caches = data.caches; @@ -306,7 +317,7 @@ export class ModalImportModels { } this.$scope.$watch('importCommon.action', this._fillCommonCachesOrTemplates(this.$scope.importCommon), true); this.$scope.importCommon.action = IMPORT_DM_NEW_CACHE; - }).subscribe(); + })).subscribe(); // New this.loadedCaches = { diff --git a/modules/web-console/frontend/app/components/page-configure/controller.js b/modules/web-console/frontend/app/components/page-configure/controller.js index 91bdf504354be..7a83aa6304ad6 100644 --- a/modules/web-console/frontend/app/components/page-configure/controller.js +++ b/modules/web-console/frontend/app/components/page-configure/controller.js @@ -16,10 +16,8 @@ */ import get from 'lodash/get'; -import {Observable} from 'rxjs/Observable'; -import 'rxjs/add/observable/merge'; -import {combineLatest} from 'rxjs/observable/combineLatest'; -import 'rxjs/add/operator/distinctUntilChanged'; +import {Observable, combineLatest} from 'rxjs'; +import {pluck, switchMap, map} from 'rxjs/operators'; import {default as ConfigureState} from './services/ConfigureState'; import {default as ConfigSelectors} from './store/selectors'; @@ -39,9 +37,12 @@ export default class PageConfigureController { $onInit() { /** @type {Observable} */ - this.clusterID$ = this.$uiRouter.globals.params$.pluck('clusterID'); - const cluster$ = this.clusterID$.switchMap((id) => this.ConfigureState.state$.let(this.ConfigSelectors.selectCluster(id))); - const isNew$ = this.clusterID$.map((v) => v === 'new'); + this.clusterID$ = this.$uiRouter.globals.params$.pipe(pluck('clusterID')); + + const cluster$ = this.clusterID$.pipe(switchMap((id) => this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCluster(id)))); + + const isNew$ = this.clusterID$.pipe(map((v) => v === 'new')); + this.clusterName$ = combineLatest(cluster$, isNew$, (cluster, isNew) => { return `${isNew ? 'Create' : 'Edit'} cluster configuration ${isNew ? '' : `‘${get(cluster, 'name')}’`}`; }); diff --git a/modules/web-console/frontend/app/components/page-configure/index.d.ts b/modules/web-console/frontend/app/components/page-configure/index.d.ts index 96773faab159a..c0eca6a93ea42 100644 --- a/modules/web-console/frontend/app/components/page-configure/index.d.ts +++ b/modules/web-console/frontend/app/components/page-configure/index.d.ts @@ -15,11 +15,10 @@ * limitations under the License. */ -import {Observable} from 'rxjs/Observable' /// declare namespace ig { - type menu = Array<{value: T, label: string}> + type menu < T > = Array<{value: T, label: string}>; namespace config { namespace formFieldSize { @@ -27,11 +26,12 @@ declare namespace ig { label: string, value: number } - type ISizeType = Array + type ISizeType = Array; interface ISizeTypes { [name: string]: ISizeType } } + namespace cluster { export type DiscoveryKinds = 'Vm' | 'Multicast' @@ -41,14 +41,14 @@ declare namespace ig { | 'Jdbc' | 'SharedFs' | 'ZooKeeper' - | 'Kubernetes' + | 'Kubernetes'; export type LoadBalancingKinds = 'RoundRobin' | 'Adaptive' | 'WeightedRandom' - | 'Custom' + | 'Custom'; - export type FailoverSPIs = 'JobStealing' | 'Never' | 'Always' | 'Custom' + export type FailoverSPIs = 'JobStealing' | 'Never' | 'Always' | 'Custom'; export interface ShortCluster { _id: string, @@ -59,9 +59,10 @@ declare namespace ig { igfs: number } } + namespace cache { - type CacheModes = 'PARTITIONED' | 'REPLICATED' | 'LOCAL' - type AtomicityModes = 'ATOMIC' | 'TRANSACTIONAL' + type CacheModes = 'PARTITIONED' | 'REPLICATED' | 'LOCAL'; + type AtomicityModes = 'ATOMIC' | 'TRANSACTIONAL'; export interface ShortCache { _id: string, cacheMode: CacheModes, @@ -69,9 +70,10 @@ declare namespace ig { backups: number } } + namespace model { - type QueryMetadataTypes = 'Annotations' | 'Configuration' - type DomainModelKinds = 'query' | 'store' | 'both' + type QueryMetadataTypes = 'Annotations' | 'Configuration'; + type DomainModelKinds = 'query' | 'store' | 'both'; export interface KeyField { databaseFieldName: string, databaseFieldType: string, @@ -92,7 +94,7 @@ declare namespace ig { field: string, alias: string } - type IndexTypes = 'SORTED' | 'FULLTEXT' | 'GEOSPATIAL' + type IndexTypes = 'SORTED' | 'FULLTEXT' | 'GEOSPATIAL'; export interface IndexField { _id: string, name?: string, @@ -135,8 +137,9 @@ declare namespace ig { hasIndex: boolean } } + namespace igfs { - type DefaultModes = 'PRIMARY' | 'PROXY' | 'DUAL_SYNC' | 'DUAL_ASYNC' + type DefaultModes = 'PRIMARY' | 'PROXY' | 'DUAL_SYNC' | 'DUAL_ASYNC'; export interface ShortIGFS { _id: string, name: string, @@ -148,4 +151,4 @@ declare namespace ig { } export as namespace ig -export = ig \ No newline at end of file +export = ig diff --git a/modules/web-console/frontend/app/components/page-configure/index.ts b/modules/web-console/frontend/app/components/page-configure/index.ts index dd2ad205dbd65..a2f76e13582b7 100644 --- a/modules/web-console/frontend/app/components/page-configure/index.ts +++ b/modules/web-console/frontend/app/components/page-configure/index.ts @@ -49,23 +49,15 @@ import pcSplitButton from './components/pc-split-button'; import {errorState} from './transitionHooks/errorState'; import {default as ActivitiesData} from 'app/core/activities/Activities.data'; -import 'rxjs/add/operator/withLatestFrom'; -import 'rxjs/add/operator/skip'; - -import {Observable} from 'rxjs/Observable'; +import {withLatestFrom, tap, filter, scan} from 'rxjs/operators'; import {navigationMenuItem, AppStore} from '../../store'; import {default as configurationIcon} from './icons/configuration.icon.svg'; import {default as IconsService} from '../ignite-icon/service'; -Observable.prototype.debug = function(l) { - return this.do((v) => console.log(l, v), (e) => console.error(l, e), () => console.log(l, 'completed')); -}; - import { editReducer2, shortObjectsReducer, - reducer, editReducer, loadingReducer, itemsEditReducerFactory, @@ -83,7 +75,6 @@ import { refsReducer } from './reducer'; -import {reducer as reduxDevtoolsReducer, devTools} from './reduxDevtoolsIntegration'; import {registerStates} from './states'; /** @@ -156,18 +147,19 @@ export default angular } }); - const la = ConfigureState.actions$.scan((acc, action) => [...acc, action], []); + const la = ConfigureState.actions$.pipe(scan((acc, action) => [...acc, action], [])); - ConfigureState.actions$ - .filter((a) => a.type === 'UNDO_ACTIONS') - .withLatestFrom(la, ({actions}, actionsWindow, initialState) => { + ConfigureState.actions$.pipe( + filter((a) => a.type === 'UNDO_ACTIONS'), + withLatestFrom(la, ({actions}, actionsWindow, initialState) => { return { type: 'APPLY_ACTIONS_UNDO', state: actionsWindow.filter((a) => !actions.includes(a)).reduce(ConfigureState._combinedReducer, {}) }; - }) - .do((a) => ConfigureState.dispatchAction(a)) - .subscribe(); + }), + tap((a) => ConfigureState.dispatchAction(a)) + ) + .subscribe(); ConfigEffects.connect(); store.dispatch(navigationMenuItem({ diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js index 7e2df806f9d05..cda9965e20dad 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js +++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js @@ -15,7 +15,8 @@ * limitations under the License. */ -import {of} from 'rxjs/observable/of'; +import {of} from 'rxjs'; +import {switchMap, catchError} from 'rxjs/operators'; import {Confirm} from 'app/services/Confirm.service'; import {DiffPatcher} from 'jsondiffpatch'; import {html} from 'jsondiffpatch/public/build/jsondiffpatch-formatters.js'; @@ -92,9 +93,9 @@ export default class ConfigChangesGuard { if (!a && !b) return Promise.resolve(true); - return of(this._hasChanges(a, b)) - .switchMap((changes) => changes ? this._confirm(changes).then(() => true) : of(true)) - .catch(() => of(false)) - .toPromise(); + return of(this._hasChanges(a, b)).pipe( + switchMap((changes) => changes ? this._confirm(changes).then(() => true) : of(true)), + catchError(() => of(false)) + ).toPromise(); } } diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigSelectionManager.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigSelectionManager.js index 243302a1995a3..b5b46d6bb6c2c 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/ConfigSelectionManager.js +++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigSelectionManager.js @@ -15,12 +15,9 @@ * limitations under the License. */ -import {Observable} from 'rxjs/Observable'; -import {merge} from 'rxjs/observable/merge'; +import {Observable, merge} from 'rxjs'; +import {share, distinctUntilChanged, startWith, filter, map, pluck, withLatestFrom, mapTo} from 'rxjs/operators'; import {RejectType} from '@uirouter/angularjs'; -import 'rxjs/add/operator/share'; -import 'rxjs/add/operator/mapTo'; -import 'rxjs/add/operator/startWith'; import isEqual from 'lodash/isEqual'; /** @@ -35,57 +32,63 @@ export default function configSelectionManager($transitions) { // but decides to stay after screen asks for leave confirmation. const abortedTransitions$ = Observable.create((observer) => { return $transitions.onError({}, (t) => observer.next(t)); - }) - .filter((t) => t.error().type === RejectType.ABORTED); + }).pipe(filter((t) => t.error().type === RejectType.ABORTED)); - const firstItemID$ = visibleRows$.withLatestFrom(itemID$, loadedItems$) - .filter(([rows, id, items]) => !id && rows && rows.length === items.length) - .pluck('0', '0', 'entity', '_id'); + const firstItemID$ = visibleRows$.pipe( + withLatestFrom(itemID$, loadedItems$), + filter(([rows, id, items]) => !id && rows && rows.length === items.length), + pluck('0', '0', 'entity', '_id') + ); - const selectedItemRowsIDs$ = selectedItemRows$.map((rows) => rows.map((r) => r._id)).share(); - const singleSelectionEdit$ = selectedItemRows$.filter((r) => r && r.length === 1).pluck('0', '_id'); - const selectedMultipleOrNone$ = selectedItemRows$.filter((r) => r.length > 1 || r.length === 0); - const loadedItemIDs$ = loadedItems$.map((rows) => new Set(rows.map((r) => r._id))).share(); - const currentItemWasRemoved$ = loadedItemIDs$ - .withLatestFrom( - itemID$.filter((v) => v && v !== 'new'), + const selectedItemRowsIDs$ = selectedItemRows$.pipe(map((rows) => rows.map((r) => r._id)), share()); + const singleSelectionEdit$ = selectedItemRows$.pipe(filter((r) => r && r.length === 1), pluck('0', '_id')); + const selectedMultipleOrNone$ = selectedItemRows$.pipe(filter((r) => r.length > 1 || r.length === 0)); + const loadedItemIDs$ = loadedItems$.pipe(map((rows) => new Set(rows.map((r) => r._id))), share()); + const currentItemWasRemoved$ = loadedItemIDs$.pipe( + withLatestFrom( + itemID$.pipe(filter((v) => v && v !== 'new')), /** * Without startWith currentItemWasRemoved$ won't emit in the following scenario: * 1. User opens items page (no item id in location). * 2. Selection manager commands to edit first item. * 3. User removes said item. */ - selectedItemRowsIDs$.startWith([]) - ) - .filter(([existingIDs, itemID, selectedIDs]) => !existingIDs.has(itemID)) - .map(([existingIDs, itemID, selectedIDs]) => selectedIDs.filter((id) => id !== itemID)) - .share(); + selectedItemRowsIDs$.pipe(startWith([])) + ), + filter(([existingIDs, itemID, selectedIDs]) => !existingIDs.has(itemID)), + map(([existingIDs, itemID, selectedIDs]) => selectedIDs.filter((id) => id !== itemID)), + share() + ); // Edit first loaded item or when there's only one item selected - const editGoes$ = merge(firstItemID$, singleSelectionEdit$) + const editGoes$ = merge(firstItemID$, singleSelectionEdit$).pipe( // Don't go to non-existing items. // Happens when user naviagtes to older history and some items were already removed. - .withLatestFrom(loadedItemIDs$).filter(([id, loaded]) => id && loaded.has(id)).pluck('0'); + withLatestFrom(loadedItemIDs$), + filter(([id, loaded]) => id && loaded.has(id)), + pluck('0') + ); // Stop edit when multiple or none items are selected or when current item was removed const editLeaves$ = merge( - selectedMultipleOrNone$.mapTo({}), - currentItemWasRemoved$.mapTo({location: 'replace', custom: {justIDUpdate: true}}) - ).share(); + selectedMultipleOrNone$.pipe(mapTo({})), + currentItemWasRemoved$.pipe(mapTo({location: 'replace', custom: {justIDUpdate: true}})) + ).pipe(share()); const selectedItemIDs$ = merge( // Select nothing when creating an item or select current item - itemID$.filter((id) => id).map((id) => id === 'new' ? [] : [id]), + itemID$.pipe(filter((id) => id), map((id) => id === 'new' ? [] : [id])), // Restore previous item selection when transition gets aborted - abortedTransitions$.withLatestFrom(itemID$, (_, id) => [id]), + abortedTransitions$.pipe(withLatestFrom(itemID$, (_, id) => [id])), // Select all incoming selected rows selectedItemRowsIDs$ - ) - // If nothing's selected and there are zero rows, ui-grid will behave as if all rows are selected - .startWith([]) - // Some scenarios cause same item to be selected multiple times in a row, - // so it makes sense to filter out duplicate entries - .distinctUntilChanged(isEqual) - .share(); + ).pipe( + // If nothing's selected and there are zero rows, ui-grid will behave as if all rows are selected + startWith([]), + // Some scenarios cause same item to be selected multiple times in a row, + // so it makes sense to filter out duplicate entries + distinctUntilChanged(isEqual), + share() + ); return {selectedItemIDs$, editGoes$, editLeaves$}; }; diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js index ed911beb08381..143944cf22a13 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js +++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js @@ -15,10 +15,8 @@ * limitations under the License. */ -import {Subject} from 'rxjs/Subject'; -import {BehaviorSubject} from 'rxjs/BehaviorSubject'; -import 'rxjs/add/operator/do'; -import 'rxjs/add/operator/scan'; +import {Subject, BehaviorSubject} from 'rxjs'; +import {tap, scan} from 'rxjs/operators'; export default class ConfigureState { constructor() { @@ -38,7 +36,10 @@ export default class ConfigureState { } }; - this.actions$.scan(reducer, {}).do((v) => this.state$.next(v)).subscribe(); + this.actions$.pipe( + scan(reducer, {}), + tap((v) => this.state$.next(v)) + ).subscribe(); } addReducer(combineFn) { diff --git a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js index d81921ea013be..5eaedde6ca1ad 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js +++ b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js @@ -15,21 +15,11 @@ * limitations under the License. */ -import {Observable} from 'rxjs/Observable'; -import 'rxjs/add/operator/filter'; -import 'rxjs/add/operator/map'; -import 'rxjs/add/operator/take'; -import 'rxjs/add/operator/switchMap'; -import 'rxjs/add/operator/merge'; -import 'rxjs/add/operator/catch'; -import 'rxjs/add/operator/withLatestFrom'; -import 'rxjs/add/observable/empty'; -import 'rxjs/add/observable/of'; -import 'rxjs/add/observable/from'; -import 'rxjs/add/observable/forkJoin'; -import 'rxjs/add/observable/timer'; import cloneDeep from 'lodash/cloneDeep'; +import {merge, timer} from 'rxjs'; +import {take, tap, ignoreElements, filter, map, pluck} from 'rxjs/operators'; + import { ofType } from '../store/effects'; @@ -50,20 +40,25 @@ export default class PageConfigure { } getClusterConfiguration({clusterID, isDemo}) { - return Observable.merge( - Observable - .timer(1) - .take(1) - .do(() => this.ConfigureState.dispatchAction({type: 'LOAD_COMPLETE_CONFIGURATION', clusterID, isDemo})) - .ignoreElements(), - this.ConfigureState.actions$.let(ofType('LOAD_COMPLETE_CONFIGURATION_ERR')).take(1).pluck('error').map((e) => Promise.reject(e)), - this.ConfigureState.state$ - .let(this.ConfigSelectors.selectCompleteClusterConfiguration({clusterID, isDemo})) - .filter((c) => c.__isComplete) - .take(1) - .map((data) => ({...data, clusters: [cloneDeep(data.cluster)]})) - ) - .take(1) + return merge( + timer(1).pipe( + take(1), + tap(() => this.ConfigureState.dispatchAction({type: 'LOAD_COMPLETE_CONFIGURATION', clusterID, isDemo})), + ignoreElements() + ), + this.ConfigureState.actions$.pipe( + ofType('LOAD_COMPLETE_CONFIGURATION_ERR'), + take(1), + pluck('error'), + map((e) => Promise.reject(e)) + ), + this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectCompleteClusterConfiguration({clusterID, isDemo}), + filter((c) => c.__isComplete), + take(1), + map((data) => ({...data, clusters: [cloneDeep(data.cluster)]})) + ) + ).pipe(take(1)) .toPromise(); } } diff --git a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.spec.js b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.spec.js index bc72cd338ea9a..145eed67f69bd 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.spec.js +++ b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.spec.js @@ -18,11 +18,8 @@ import {suite, test} from 'mocha'; import {assert} from 'chai'; import {spy} from 'sinon'; -import {TestScheduler} from 'rxjs/testing/TestScheduler'; -import {Observable} from 'rxjs/Observable'; - -import 'rxjs/add/observable/of'; -import 'rxjs/add/observable/throw'; +import {of, throwError} from 'rxjs'; +import {TestScheduler} from 'rxjs/testing'; const mocks = () => new Map([ ['IgniteConfigurationResource', {}], @@ -79,7 +76,7 @@ suite.skip('PageConfigure service', () => { const deps = mocks() .set('Clusters', { - saveCluster$: (c) => Observable.of({data: 99}) + saveCluster$: (c) => of({data: 99}) }) .set('ConfigureState', { actions$: testScheduler.createHotObservable(actions, values), @@ -135,8 +132,8 @@ suite.skip('PageConfigure service', () => { const deps = mocks() .set('Clusters', { saveCluster$: (c) => c.name === values.b.clusters[0].name - ? Observable.of({data: 99}) - : Observable.throw() + ? of({data: 99}) + : throwError() }) .set('ConfigureState', { actions$: testScheduler.createHotObservable(actions, values), @@ -188,7 +185,7 @@ suite.skip('PageConfigure service', () => { dispatchAction: spy() }) .set('Clusters', { - removeCluster$: (v) => Observable.of(v) + removeCluster$: (v) => of(v) }); const s = new PageConfigure(...deps.values()); @@ -232,7 +229,7 @@ suite.skip('PageConfigure service', () => { dispatchAction: spy() }) .set('Clusters', { - removeCluster$: (v) => v._id % 2 ? Observable.of(v) : Observable.throw() + removeCluster$: (v) => v._id % 2 ? of(v) : throwError() }); const s = new PageConfigure(...deps.values()); diff --git a/modules/web-console/frontend/app/components/page-configure/states.js b/modules/web-console/frontend/app/components/page-configure/states.js index 3ba5bb70f18bc..3b8dec948d644 100644 --- a/modules/web-console/frontend/app/components/page-configure/states.js +++ b/modules/web-console/frontend/app/components/page-configure/states.js @@ -20,19 +20,21 @@ import pageConfigureAdvancedModelsComponent from '../page-configure-advanced/com import pageConfigureAdvancedCachesComponent from '../page-configure-advanced/components/page-configure-advanced-caches/component'; import pageConfigureAdvancedIGFSComponent from '../page-configure-advanced/components/page-configure-advanced-igfs/component'; -import {Observable} from 'rxjs/Observable'; +import {Observable, from, combineLatest} from 'rxjs'; +import {switchMap, take, map} from 'rxjs/operators'; const idRegex = `new|[a-z0-9]+`; const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', function(ConfigSelectors, ConfigureState, {etp}, $transition$) { if ($transition$.params().clusterID === 'new') return Promise.resolve(); - return Observable.fromPromise($transition$.injector().getAsync('_cluster')) - .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1)) - .switchMap((cluster) => { + return from($transition$.injector().getAsync('_cluster')).pipe( + switchMap(() => ConfigureState.state$.pipe(ConfigSelectors.selectCluster($transition$.params().clusterID), take(1))), + switchMap((cluster) => { return etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}); }) - .toPromise(); + ) + .toPromise(); }]; function registerStates($stateProvider) { @@ -77,17 +79,19 @@ function registerStates($stateProvider) { redirectTo: ($transition$) => { const [ConfigureState, ConfigSelectors] = ['ConfigureState', 'ConfigSelectors'].map((t) => $transition$.injector().get(t)); const waitFor = ['_cluster', '_shortClusters'].map((t) => $transition$.injector().getAsync(t)); - return Observable.fromPromise(Promise.all(waitFor)).switchMap(() => { - return Observable.combineLatest( - ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1), - ConfigureState.state$.let(ConfigSelectors.selectShortClusters()).take(1) - ); - }) - .map(([cluster = {caches: []}, clusters]) => { - return (clusters.value.size > 10 || cluster.caches.length > 5) - ? 'base.configuration.edit.advanced' - : 'base.configuration.edit.basic'; - }) + return from(Promise.all(waitFor)).pipe( + switchMap(() => { + return combineLatest( + ConfigureState.state$.pipe(ConfigSelectors.selectCluster($transition$.params().clusterID), take(1)), + ConfigureState.state$.pipe(ConfigSelectors.selectShortClusters(), take(1)) + ); + }), + map(([cluster = {caches: []}, clusters]) => { + return (clusters.value.size > 10 || cluster.caches.length > 5) + ? 'base.configuration.edit.advanced' + : 'base.configuration.edit.basic'; + }) + ) .toPromise(); }, failState: 'signin', @@ -138,16 +142,17 @@ function registerStates($stateProvider) { if ($transition$.params().clusterID === 'new') return Promise.resolve(); - return Observable.fromPromise($transition$.injector().getAsync('_cluster')) - .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1)) - .map((cluster) => { + return from($transition$.injector().getAsync('_cluster')).pipe( + switchMap(() => ConfigureState.state$.pipe(ConfigSelectors.selectCluster($transition$.params().clusterID), take(1))), + map((cluster) => { return Promise.all([ etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}), etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}), etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id}) ]); }) - .toPromise(); + ) + .toPromise(); }] }, resolvePolicy: { @@ -163,6 +168,7 @@ function registerStates($stateProvider) { resolve: { _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => { const {clusterID, cacheID} = $transition$.params(); + if (cacheID === 'new') return Promise.resolve(); @@ -188,15 +194,15 @@ function registerStates($stateProvider) { if ($transition$.params().clusterID === 'new') return Promise.resolve(); - return Observable.fromPromise($transition$.injector().getAsync('_cluster')) - .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1)) - .map((cluster) => { + return from($transition$.injector().getAsync('_cluster')).pipe( + switchMap(() => ConfigureState.state$.pipe(ConfigSelectors.selectCluster($transition$.params().clusterID), take(1))), + map((cluster) => { return Promise.all([ etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}), etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}) ]); }) - .toPromise(); + ).toPromise(); }] }, resolvePolicy: { @@ -235,14 +241,14 @@ function registerStates($stateProvider) { if ($transition$.params().clusterID === 'new') return Promise.resolve(); - return Observable.fromPromise($transition$.injector().getAsync('_cluster')) - .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1)) - .map((cluster) => { + return from($transition$.injector().getAsync('_cluster')).pipe( + switchMap(() => ConfigureState.state$.pipe(ConfigSelectors.selectCluster($transition$.params().clusterID), take(1))), + map((cluster) => { return Promise.all([ etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id}) ]); }) - .toPromise(); + ).toPromise(); }] }, resolvePolicy: { diff --git a/modules/web-console/frontend/app/components/page-configure/store/effects.js b/modules/web-console/frontend/app/components/page-configure/store/effects.js index 014076b2b5a76..ac807dce95d7d 100644 --- a/modules/web-console/frontend/app/components/page-configure/store/effects.js +++ b/modules/web-console/frontend/app/components/page-configure/store/effects.js @@ -15,23 +15,8 @@ * limitations under the License. */ -import {Observable} from 'rxjs/Observable'; -import 'rxjs/add/operator/ignoreElements'; -import 'rxjs/add/operator/let'; -import 'rxjs/add/operator/exhaustMap'; -import 'rxjs/add/operator/switchMap'; -import 'rxjs/add/operator/map'; -import 'rxjs/add/operator/pluck'; -import 'rxjs/add/operator/withLatestFrom'; -import 'rxjs/add/operator/merge'; -import 'rxjs/add/operator/take'; -import 'rxjs/add/operator/catch'; -import 'rxjs/add/operator/zip'; -import {merge} from 'rxjs/observable/merge'; -import {empty} from 'rxjs/observable/empty'; -import {of} from 'rxjs/observable/of'; -import {from} from 'rxjs/observable/from'; -import {fromPromise} from 'rxjs/observable/fromPromise'; +import {merge, empty, of, from} from 'rxjs'; +import {mapTo, filter, tap, ignoreElements, exhaustMap, switchMap, map, pluck, withLatestFrom, take, catchError, zip} from 'rxjs/operators'; import uniq from 'lodash/uniq'; import {uniqueName} from 'app/utils/uniqueName'; import {defaultNames} from '../defaultNames'; @@ -81,7 +66,7 @@ import Models from 'app/services/Models'; import IGFSs from 'app/services/IGFSs'; import {Confirm} from 'app/services/Confirm.service'; -export const ofType = (type) => (s) => s.filter((a) => a.type === type); +export const ofType = (type) => (s) => s.pipe(filter((a) => a.type === type)); export default class ConfigEffects { static $inject = [ @@ -124,35 +109,37 @@ export default class ConfigEffects { this.Confirm = Confirm; this.configurationDownload = ConfigurationDownload; - this.loadConfigurationEffect$ = this.ConfigureState.actions$ - .let(ofType('LOAD_COMPLETE_CONFIGURATION')) - .exhaustMap((action) => { - return fromPromise(this.Clusters.getConfiguration(action.clusterID)) - .switchMap(({data}) => of( + this.loadConfigurationEffect$ = this.ConfigureState.actions$.pipe( + ofType('LOAD_COMPLETE_CONFIGURATION'), + exhaustMap((action) => { + return from(this.Clusters.getConfiguration(action.clusterID)).pipe( + switchMap(({data}) => of( completeConfiguration(data), {type: 'LOAD_COMPLETE_CONFIGURATION_OK', data} - )) - .catch((error) => of({ + )), + catchError((error) => of({ type: 'LOAD_COMPLETE_CONFIGURATION_ERR', error: { message: `Failed to load cluster configuration: ${error.data}.` }, action - })); - }); + }))); + }) + ); - this.storeConfigurationEffect$ = this.ConfigureState.actions$ - .let(ofType(COMPLETE_CONFIGURATION)) - .exhaustMap(({configuration: {cluster, caches, models, igfss}}) => of(...[ + this.storeConfigurationEffect$ = this.ConfigureState.actions$.pipe( + ofType(COMPLETE_CONFIGURATION), + exhaustMap(({configuration: {cluster, caches, models, igfss}}) => of(...[ cluster && {type: clustersActionTypes.UPSERT, items: [cluster]}, caches && caches.length && {type: cachesActionTypes.UPSERT, items: caches}, models && models.length && {type: modelsActionTypes.UPSERT, items: models}, igfss && igfss.length && {type: igfssActionTypes.UPSERT, items: igfss} - ].filter((v) => v))); + ].filter((v) => v))) + ); - this.saveCompleteConfigurationEffect$ = this.ConfigureState.actions$ - .let(ofType(ADVANCED_SAVE_COMPLETE_CONFIGURATION)) - .switchMap((action) => { + this.saveCompleteConfigurationEffect$ = this.ConfigureState.actions$.pipe( + ofType(ADVANCED_SAVE_COMPLETE_CONFIGURATION), + switchMap((action) => { const actions = [ { type: modelsActionTypes.UPSERT, @@ -188,62 +175,68 @@ export default class ConfigEffects { } ].filter((a) => a.items.length); - return of(...actions) - .merge( - fromPromise(Clusters.saveAdvanced(action.changedItems)) - .switchMap((res) => { - return of( - {type: 'EDIT_CLUSTER', cluster: action.changedItems.cluster}, - {type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK', changedItems: action.changedItems} - ); - }) - .catch((res) => { - return of({ - type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_ERR', - changedItems: action.changedItems, - action, - error: { - message: `Failed to save cluster "${action.changedItems.cluster.name}": ${res.data}.` - } - }, { - type: 'UNDO_ACTIONS', - actions - }); - }) + return merge( + of(...actions), + from(Clusters.saveAdvanced(action.changedItems)).pipe( + switchMap((res) => { + return of( + {type: 'EDIT_CLUSTER', cluster: action.changedItems.cluster}, + {type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK', changedItems: action.changedItems} + ); + }), + catchError((res) => { + return of({ + type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_ERR', + changedItems: action.changedItems, + action, + error: { + message: `Failed to save cluster "${action.changedItems.cluster.name}": ${res.data}.` + } + }, { + type: 'UNDO_ACTIONS', + actions + }); + }) + ) ); - }); - - this.addCacheToEditEffect$ = this.ConfigureState.actions$ - .let(ofType('ADD_CACHE_TO_EDIT')) - .switchMap(() => this.ConfigureState.state$.let(this.ConfigSelectors.selectCacheToEdit('new')).take(1)) - .map((cache) => ({type: 'UPSERT_CLUSTER_ITEM', itemType: 'caches', item: cache})); - - this.errorNotificationsEffect$ = this.ConfigureState.actions$ - .filter((a) => a.error) - .do((action) => this.IgniteMessages.showError(action.error)) - .ignoreElements(); - - this.loadUserClustersEffect$ = this.ConfigureState.actions$ - .let(ofType('LOAD_USER_CLUSTERS')) - .exhaustMap((a) => { - return fromPromise(this.Clusters.getClustersOverview()) - .switchMap(({data}) => of( + }) + ); + + this.addCacheToEditEffect$ = this.ConfigureState.actions$.pipe( + ofType('ADD_CACHE_TO_EDIT'), + switchMap(() => this.ConfigureState.state$.pipe(this.ConfigSelectors.selectCacheToEdit('new'), take(1))), + map((cache) => ({type: 'UPSERT_CLUSTER_ITEM', itemType: 'caches', item: cache})) + ); + + this.errorNotificationsEffect$ = this.ConfigureState.actions$.pipe( + filter((a) => a.error), + tap((action) => this.IgniteMessages.showError(action.error)), + ignoreElements() + ); + + this.loadUserClustersEffect$ = this.ConfigureState.actions$.pipe( + ofType('LOAD_USER_CLUSTERS'), + exhaustMap((a) => { + return from(this.Clusters.getClustersOverview()).pipe( + switchMap(({data}) => of( {type: shortClustersActionTypes.SET, items: data}, {type: `${a.type}_OK`} - )) - .catch((error) => of({ + )), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load clusters: ${error.data}` }, action: a - })); - }); + })) + ); + }) + ); - this.loadAndEditClusterEffect$ = ConfigureState.actions$ - .let(ofType('LOAD_AND_EDIT_CLUSTER')) - .withLatestFrom(this.ConfigureState.state$.let(this.ConfigSelectors.selectShortClustersValue())) - .exhaustMap(([a, shortClusters]) => { + this.loadAndEditClusterEffect$ = ConfigureState.actions$.pipe( + ofType('LOAD_AND_EDIT_CLUSTER'), + withLatestFrom(this.ConfigureState.state$.pipe(this.ConfigSelectors.selectShortClustersValue())), + exhaustMap(([a, shortClusters]) => { if (a.clusterID === 'new') { return of( { @@ -256,226 +249,268 @@ export default class ConfigEffects { {type: 'LOAD_AND_EDIT_CLUSTER_OK'} ); } - return this.ConfigureState.state$.let(this.ConfigSelectors.selectCluster(a.clusterID)).take(1) - .switchMap((cluster) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectCluster(a.clusterID), + take(1), + switchMap((cluster) => { if (cluster) { return of( {type: 'EDIT_CLUSTER', cluster}, {type: 'LOAD_AND_EDIT_CLUSTER_OK'} ); } - return from(this.Clusters.getCluster(a.clusterID)) - .switchMap(({data}) => of( + return from(this.Clusters.getCluster(a.clusterID)).pipe( + switchMap(({data}) => of( {type: clustersActionTypes.UPSERT, items: [data]}, {type: 'EDIT_CLUSTER', cluster: data}, {type: 'LOAD_AND_EDIT_CLUSTER_OK'} - )) - .catch((error) => of({ + )), + catchError((error) => of({ type: 'LOAD_AND_EDIT_CLUSTER_ERR', error: { message: `Failed to load cluster: ${error.data}.` } - })); - }); - }); - - this.loadCacheEffect$ = this.ConfigureState.actions$ - .let(ofType('LOAD_CACHE')) - .exhaustMap((a) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectCache(a.cacheID)).take(1) - .switchMap((cache) => { + })) + ); + }) + ); + }) + ); + + this.loadCacheEffect$ = this.ConfigureState.actions$.pipe( + ofType('LOAD_CACHE'), + exhaustMap((a) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectCache(a.cacheID), + take(1), + switchMap((cache) => { if (cache) return of({type: `${a.type}_OK`, cache}); - return fromPromise(this.Caches.getCache(a.cacheID)) - .switchMap(({data}) => of( + return from(this.Caches.getCache(a.cacheID)).pipe( + switchMap(({data}) => of( {type: 'CACHE', cache: data}, {type: `${a.type}_OK`, cache: data} - )); - }) - .catch((error) => of({ + )) + ); + }), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load cache: ${error.data}.` } - })); - }); + })) + ); + }) + ); - this.storeCacheEffect$ = this.ConfigureState.actions$ - .let(ofType('CACHE')) - .map((a) => ({type: cachesActionTypes.UPSERT, items: [a.cache]})); + this.storeCacheEffect$ = this.ConfigureState.actions$.pipe( + ofType('CACHE'), + map((a) => ({type: cachesActionTypes.UPSERT, items: [a.cache]})) + ); - this.loadShortCachesEffect$ = ConfigureState.actions$ - .let(ofType('LOAD_SHORT_CACHES')) - .exhaustMap((a) => { + this.loadShortCachesEffect$ = ConfigureState.actions$.pipe( + ofType('LOAD_SHORT_CACHES'), + exhaustMap((a) => { if (!(a.ids || []).length) return of({type: `${a.type}_OK`}); - return this.ConfigureState.state$.let(this.ConfigSelectors.selectShortCaches()).take(1) - .switchMap((items) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectShortCaches(), + take(1), + switchMap((items) => { if (!items.pristine && a.ids && a.ids.every((_id) => items.value.has(_id))) return of({type: `${a.type}_OK`}); - return fromPromise(this.Clusters.getClusterCaches(a.clusterID)) - .switchMap(({data}) => of( + return from(this.Clusters.getClusterCaches(a.clusterID)).pipe( + switchMap(({data}) => of( {type: shortCachesActionTypes.UPSERT, items: data}, {type: `${a.type}_OK`} - )); - }) - .catch((error) => of({ + )) + ); + }), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load caches: ${error.data}.` }, action: a - })); - }); - - this.loadIgfsEffect$ = this.ConfigureState.actions$ - .let(ofType('LOAD_IGFS')) - .exhaustMap((a) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectIGFS(a.igfsID)).take(1) - .switchMap((igfs) => { + })) + ); + }) + ); + + this.loadIgfsEffect$ = this.ConfigureState.actions$.pipe( + ofType('LOAD_IGFS'), + exhaustMap((a) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectIGFS(a.igfsID), + take(1), + switchMap((igfs) => { if (igfs) return of({type: `${a.type}_OK`, igfs}); - return fromPromise(this.IGFSs.getIGFS(a.igfsID)) - .switchMap(({data}) => of( + return from(this.IGFSs.getIGFS(a.igfsID)).pipe( + switchMap(({data}) => of( {type: 'IGFS', igfs: data}, {type: `${a.type}_OK`, igfs: data} - )); - }) - .catch((error) => of({ + )) + ); + }), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load IGFS: ${error.data}.` } - })); - }); + })) + ); + }) + ); - this.storeIgfsEffect$ = this.ConfigureState.actions$ - .let(ofType('IGFS')) - .map((a) => ({type: igfssActionTypes.UPSERT, items: [a.igfs]})); + this.storeIgfsEffect$ = this.ConfigureState.actions$.pipe( + ofType('IGFS'), + map((a) => ({type: igfssActionTypes.UPSERT, items: [a.igfs]})) + ); - this.loadShortIgfssEffect$ = ConfigureState.actions$ - .let(ofType('LOAD_SHORT_IGFSS')) - .exhaustMap((a) => { + this.loadShortIgfssEffect$ = ConfigureState.actions$.pipe( + ofType('LOAD_SHORT_IGFSS'), + exhaustMap((a) => { if (!(a.ids || []).length) { return of( {type: shortIGFSsActionTypes.UPSERT, items: []}, {type: `${a.type}_OK`} ); } - return this.ConfigureState.state$.let(this.ConfigSelectors.selectShortIGFSs()).take(1) - .switchMap((items) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectShortIGFSs(), + take(1), + switchMap((items) => { if (!items.pristine && a.ids && a.ids.every((_id) => items.value.has(_id))) return of({type: `${a.type}_OK`}); - return fromPromise(this.Clusters.getClusterIGFSs(a.clusterID)) - .switchMap(({data}) => of( + return from(this.Clusters.getClusterIGFSs(a.clusterID)).pipe( + switchMap(({data}) => of( {type: shortIGFSsActionTypes.UPSERT, items: data}, {type: `${a.type}_OK`} - )); - }) - .catch((error) => of({ + )) + ); + }), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load IGFSs: ${error.data}.` }, action: a - })); - }); - - this.loadModelEffect$ = this.ConfigureState.actions$ - .let(ofType('LOAD_MODEL')) - .exhaustMap((a) => { - return this.ConfigureState.state$.let(this.ConfigSelectors.selectModel(a.modelID)).take(1) - .switchMap((model) => { + })) + ); + }) + ); + + this.loadModelEffect$ = this.ConfigureState.actions$.pipe( + ofType('LOAD_MODEL'), + exhaustMap((a) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectModel(a.modelID), + take(1), + switchMap((model) => { if (model) return of({type: `${a.type}_OK`, model}); - return fromPromise(this.Models.getModel(a.modelID)) - .switchMap(({data}) => of( + return from(this.Models.getModel(a.modelID)).pipe( + switchMap(({data}) => of( {type: 'MODEL', model: data}, {type: `${a.type}_OK`, model: data} - )); - }) - .catch((error) => of({ + )) + ); + }), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load domain model: ${error.data}.` } - })); - }); + })) + ); + }) + ); - this.storeModelEffect$ = this.ConfigureState.actions$ - .let(ofType('MODEL')) - .map((a) => ({type: modelsActionTypes.UPSERT, items: [a.model]})); + this.storeModelEffect$ = this.ConfigureState.actions$.pipe( + ofType('MODEL'), + map((a) => ({type: modelsActionTypes.UPSERT, items: [a.model]})) + ); - this.loadShortModelsEffect$ = this.ConfigureState.actions$ - .let(ofType('LOAD_SHORT_MODELS')) - .exhaustMap((a) => { + this.loadShortModelsEffect$ = this.ConfigureState.actions$.pipe( + ofType('LOAD_SHORT_MODELS'), + exhaustMap((a) => { if (!(a.ids || []).length) { return of( {type: shortModelsActionTypes.UPSERT, items: []}, {type: `${a.type}_OK`} ); } - return this.ConfigureState.state$.let(this.ConfigSelectors.selectShortModels()).take(1) - .switchMap((items) => { + return this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectShortModels(), + take(1), + switchMap((items) => { if (!items.pristine && a.ids && a.ids.every((_id) => items.value.has(_id))) return of({type: `${a.type}_OK`}); - return fromPromise(this.Clusters.getClusterModels(a.clusterID)) - .switchMap(({data}) => of( + return from(this.Clusters.getClusterModels(a.clusterID)).pipe( + switchMap(({data}) => of( {type: shortModelsActionTypes.UPSERT, items: data}, {type: `${a.type}_OK`} - )); - }) - .catch((error) => of({ + )) + ); + }), + catchError((error) => of({ type: `${a.type}_ERR`, error: { message: `Failed to load domain models: ${error.data}.` }, action: a - })); - }); - - this.basicSaveRedirectEffect$ = this.ConfigureState.actions$ - .let(ofType(BASIC_SAVE_OK)) - .do((a) => this.$state.go('base.configuration.edit.basic', {clusterID: a.changedItems.cluster._id}, {location: 'replace', custom: {justIDUpdate: true}})) - .ignoreElements(); - - this.basicDownloadAfterSaveEffect$ = this.ConfigureState.actions$.let(ofType(BASIC_SAVE_AND_DOWNLOAD)) - .zip(this.ConfigureState.actions$.let(ofType(BASIC_SAVE_OK))) - .pluck('1') - .do((a) => this.configurationDownload.downloadClusterConfiguration(a.changedItems.cluster)) - .ignoreElements(); + })) + ); + }) + ); + + this.basicSaveRedirectEffect$ = this.ConfigureState.actions$.pipe( + ofType(BASIC_SAVE_OK), + tap((a) => this.$state.go('base.configuration.edit.basic', {clusterID: a.changedItems.cluster._id}, {location: 'replace', custom: {justIDUpdate: true}})), + ignoreElements() + ); + + this.basicDownloadAfterSaveEffect$ = this.ConfigureState.actions$.pipe( + ofType(BASIC_SAVE_AND_DOWNLOAD), + zip(this.ConfigureState.actions$.pipe(ofType(BASIC_SAVE_OK))), + pluck('1'), + tap((a) => this.configurationDownload.downloadClusterConfiguration(a.changedItems.cluster)), + ignoreElements() + ); this.advancedDownloadAfterSaveEffect$ = merge( - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_CLUSTER)), - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_CACHE)), - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_IGFS)), - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_MODEL)), - ) - .filter((a) => a.download) - .zip(this.ConfigureState.actions$.let(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK'))) - .pluck('1') - .do((a) => this.configurationDownload.downloadClusterConfiguration(a.changedItems.cluster)) - .ignoreElements(); - - this.advancedSaveRedirectEffect$ = this.ConfigureState.actions$ - .let(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK')) - .withLatestFrom(this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_COMPLETE_CONFIGURATION))) - .pluck('1', 'changedItems') - .map((req) => { + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_CLUSTER)), + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_CACHE)), + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_IGFS)), + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_MODEL)), + ).pipe( + filter((a) => a.download), + zip(this.ConfigureState.actions$.pipe(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK'))), + pluck('1'), + tap((a) => this.configurationDownload.downloadClusterConfiguration(a.changedItems.cluster)), + ignoreElements() + ); + + this.advancedSaveRedirectEffect$ = this.ConfigureState.actions$.pipe( + ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK'), + withLatestFrom(this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_COMPLETE_CONFIGURATION))), + pluck('1', 'changedItems'), + map((req) => { const firstChangedItem = Object.keys(req).filter((k) => k !== 'cluster') .map((k) => Array.isArray(req[k]) ? [k, req[k][0]] : [k, req[k]]) .filter((v) => v[1]) .pop(); return firstChangedItem ? [...firstChangedItem, req.cluster] : ['cluster', req.cluster, req.cluster]; - }) - .do(([type, value, cluster]) => { + }), + tap(([type, value, cluster]) => { const go = (state, params = {}) => this.$state.go( state, {...params, clusterID: cluster._id}, {location: 'replace', custom: {justIDUpdate: true}} ); @@ -523,66 +558,76 @@ export default class ConfigEffects { default: break; } - }) - .ignoreElements(); + }), + ignoreElements() + ); - this.removeClusterItemsEffect$ = this.ConfigureState.actions$ - .let(ofType(REMOVE_CLUSTER_ITEMS)) - .exhaustMap((a) => { + this.removeClusterItemsEffect$ = this.ConfigureState.actions$.pipe( + ofType(REMOVE_CLUSTER_ITEMS), + exhaustMap((a) => { return a.confirm // TODO: list items to remove in confirmation - ? fromPromise(this.Confirm.confirm('Are you sure you want to remove these items?')) - .mapTo(a) - .catch(() => empty()) + ? from(this.Confirm.confirm('Are you sure you want to remove these items?')).pipe( + mapTo(a), + catchError(() => empty()) + ) : of(a); - }) - .map((a) => removeClusterItemsConfirmed(a.clusterID, a.itemType, a.itemIDs)); - - this.persistRemovedClusterItemsEffect$ = this.ConfigureState.actions$ - .let(ofType(REMOVE_CLUSTER_ITEMS_CONFIRMED)) - .withLatestFrom(this.ConfigureState.actions$.let(ofType(REMOVE_CLUSTER_ITEMS))) - .filter(([a, b]) => { + }), + map((a) => removeClusterItemsConfirmed(a.clusterID, a.itemType, a.itemIDs)) + ); + + this.persistRemovedClusterItemsEffect$ = this.ConfigureState.actions$.pipe( + ofType(REMOVE_CLUSTER_ITEMS_CONFIRMED), + withLatestFrom(this.ConfigureState.actions$.pipe(ofType(REMOVE_CLUSTER_ITEMS))), + filter(([a, b]) => { return a.itemType === b.itemType && b.save && JSON.stringify(a.itemIDs) === JSON.stringify(b.itemIDs); - }) - .pluck('0') - .withLatestFrom(this.ConfigureState.state$.pluck('edit')) - .map(([action, edit]) => advancedSaveCompleteConfiguration(edit)); - - this.confirmClustersRemovalEffect$ = this.ConfigureState.actions$ - .let(ofType(CONFIRM_CLUSTERS_REMOVAL)) - .pluck('clusterIDs') - .switchMap((ids) => this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterNames(ids)).take(1)) - .exhaustMap((names) => { - return fromPromise(this.Confirm.confirm(` + }), + pluck('0'), + withLatestFrom(this.ConfigureState.state$.pipe(pluck('edit'))), + map(([action, edit]) => advancedSaveCompleteConfiguration(edit)) + ); + + this.confirmClustersRemovalEffect$ = this.ConfigureState.actions$.pipe( + ofType(CONFIRM_CLUSTERS_REMOVAL), + pluck('clusterIDs'), + switchMap((ids) => this.ConfigureState.state$.pipe( + this.ConfigSelectors.selectClusterNames(ids), + take(1) + )), + exhaustMap((names) => { + return from(this.Confirm.confirm(`

Are you sure you want to remove these clusters?

    ${names.map((name) => `
  • ${name}
  • `).join('')}
- `)) - .map(confirmClustersRemovalOK) - .catch(() => Observable.empty()); - }); - - this.persistRemovedClustersLocallyEffect$ = this.ConfigureState.actions$ - .let(ofType(CONFIRM_CLUSTERS_REMOVAL_OK)) - .withLatestFrom(this.ConfigureState.actions$.let(ofType(CONFIRM_CLUSTERS_REMOVAL))) - .switchMap(([, {clusterIDs}]) => of( + `)).pipe( + map(confirmClustersRemovalOK), + catchError(() => empty()) + ); + }) + ); + + this.persistRemovedClustersLocallyEffect$ = this.ConfigureState.actions$.pipe( + ofType(CONFIRM_CLUSTERS_REMOVAL_OK), + withLatestFrom(this.ConfigureState.actions$.pipe(ofType(CONFIRM_CLUSTERS_REMOVAL))), + switchMap(([, {clusterIDs}]) => of( {type: shortClustersActionTypes.REMOVE, ids: clusterIDs}, {type: clustersActionTypes.REMOVE, ids: clusterIDs} - )); - - this.persistRemovedClustersRemotelyEffect$ = this.ConfigureState.actions$ - .let(ofType(CONFIRM_CLUSTERS_REMOVAL_OK)) - .withLatestFrom( - this.ConfigureState.actions$.let(ofType(CONFIRM_CLUSTERS_REMOVAL)), - this.ConfigureState.actions$.let(ofType(shortClustersActionTypes.REMOVE)), - this.ConfigureState.actions$.let(ofType(clustersActionTypes.REMOVE)) - ) - .switchMap(([, {clusterIDs}, ...backup]) => this.Clusters.removeCluster$(clusterIDs) - .mapTo({ + )) + ); + + this.persistRemovedClustersRemotelyEffect$ = this.ConfigureState.actions$.pipe( + ofType(CONFIRM_CLUSTERS_REMOVAL_OK), + withLatestFrom( + this.ConfigureState.actions$.pipe(ofType(CONFIRM_CLUSTERS_REMOVAL)), + this.ConfigureState.actions$.pipe(ofType(shortClustersActionTypes.REMOVE)), + this.ConfigureState.actions$.pipe(ofType(clustersActionTypes.REMOVE)) + ), + switchMap(([, {clusterIDs}, ...backup]) => this.Clusters.removeCluster$(clusterIDs).pipe( + mapTo({ type: 'REMOVE_CLUSTERS_OK' - }) - .catch((e) => of( + }), + catchError((e) => of( { type: 'REMOVE_CLUSTERS_ERR', error: { @@ -594,13 +639,15 @@ export default class ConfigEffects { actions: backup } )) - ); + )) + ); - this.notifyRemoteClustersRemoveSuccessEffect$ = this.ConfigureState.actions$ - .let(ofType('REMOVE_CLUSTERS_OK')) - .withLatestFrom(this.ConfigureState.actions$.let(ofType(CONFIRM_CLUSTERS_REMOVAL))) - .do(([, {clusterIDs}]) => this.IgniteMessages.showInfo(`Cluster(s) removed: ${clusterIDs.length}`)) - .ignoreElements(); + this.notifyRemoteClustersRemoveSuccessEffect$ = this.ConfigureState.actions$.pipe( + ofType('REMOVE_CLUSTERS_OK'), + withLatestFrom(this.ConfigureState.actions$.pipe(ofType(CONFIRM_CLUSTERS_REMOVAL))), + tap(([, {clusterIDs}]) => this.IgniteMessages.showInfo(`Cluster(s) removed: ${clusterIDs.length}`)), + ignoreElements() + ); const _applyChangedIDs = (edit, {cache, igfs, model, cluster} = {}) => ({ cluster: { @@ -616,22 +663,24 @@ export default class ConfigEffects { }); this.advancedSaveCacheEffect$ = merge( - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_CLUSTER)), - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_CACHE)), - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_IGFS)), - this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_MODEL)), - ) - .withLatestFrom(this.ConfigureState.state$.pluck('edit')) - .map(([action, edit]) => ({ + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_CLUSTER)), + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_CACHE)), + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_IGFS)), + this.ConfigureState.actions$.pipe(ofType(ADVANCED_SAVE_MODEL)), + ).pipe( + withLatestFrom(this.ConfigureState.state$.pipe(pluck('edit'))), + map(([action, edit]) => ({ type: ADVANCED_SAVE_COMPLETE_CONFIGURATION, changedItems: _applyChangedIDs(edit, action) - })); - - this.basicSaveEffect$ = this.ConfigureState.actions$ - .let(ofType(BASIC_SAVE)) - .merge(this.ConfigureState.actions$.let(ofType(BASIC_SAVE_AND_DOWNLOAD))) - .withLatestFrom(this.ConfigureState.state$.pluck('edit')) - .switchMap(([action, edit]) => { + })) + ); + + this.basicSaveEffect$ = merge( + this.ConfigureState.actions$.pipe(ofType(BASIC_SAVE)), + this.ConfigureState.actions$.pipe(ofType(BASIC_SAVE_AND_DOWNLOAD)) + ).pipe( + withLatestFrom(this.ConfigureState.state$.pipe(pluck('edit'))), + switchMap(([action, edit]) => { const changedItems = _applyChangedIDs(edit, {cluster: action.cluster}); const actions = [{ type: cachesActionTypes.UPSERT, @@ -651,24 +700,27 @@ export default class ConfigEffects { } ].filter((a) => a.items.length); - return Observable.of(...actions) - .merge( - Observable.fromPromise(this.Clusters.saveBasic(changedItems)) - .switchMap((res) => Observable.of( - {type: 'EDIT_CLUSTER', cluster: changedItems.cluster}, - basicSaveOK(changedItems) - )) - .catch((res) => Observable.of( - basicSaveErr(changedItems, res), - {type: 'UNDO_ACTIONS', actions} - )) + return merge( + of(...actions), + from(this.Clusters.saveBasic(changedItems)).pipe( + switchMap((res) => of( + {type: 'EDIT_CLUSTER', cluster: changedItems.cluster}, + basicSaveOK(changedItems) + )), + catchError((res) => of( + basicSaveErr(changedItems, res), + {type: 'UNDO_ACTIONS', actions} + )) + ) ); - }); + }) + ); - this.basicSaveOKMessagesEffect$ = this.ConfigureState.actions$ - .let(ofType(BASIC_SAVE_OK)) - .do((action) => this.IgniteMessages.showInfo(`Cluster "${action.changedItems.cluster.name}" saved.`)) - .ignoreElements(); + this.basicSaveOKMessagesEffect$ = this.ConfigureState.actions$.pipe( + ofType(BASIC_SAVE_OK), + tap((action) => this.IgniteMessages.showInfo(`Cluster "${action.changedItems.cluster.name}" saved.`)), + ignoreElements() + ); } /** @@ -691,21 +743,21 @@ export default class ConfigEffects { setTimeout(() => this.ConfigureState.dispatchAction(action)); - return this.ConfigureState.actions$ - .filter((a) => a.type === ok || a.type === err) - .take(1) - .map((a) => { + return this.ConfigureState.actions$.pipe( + filter((a) => a.type === ok || a.type === err), + take(1), + map((a) => { if (a.type === err) throw a; else return a; }) - .toPromise(); + ).toPromise(); }; connect() { return merge( ...Object.keys(this).filter((k) => k.endsWith('Effect$')).map((k) => this[k]) - ).do((a) => this.ConfigureState.dispatchAction(a)).subscribe(); + ).pipe(tap((a) => this.ConfigureState.dispatchAction(a))).subscribe(); } } diff --git a/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js b/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js index 10e066330a795..f8b39098bcbbb 100644 --- a/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js +++ b/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js @@ -16,11 +16,10 @@ */ import {assert} from 'chai'; -import {of} from 'rxjs/observable/of'; -import {_throw} from 'rxjs/observable/throw'; +import {of, throwError} from 'rxjs'; +import {TestScheduler} from 'rxjs/testing'; import {default as Effects} from './effects'; import {default as Selectors} from './selectors'; -import {TestScheduler} from 'rxjs/testing/TestScheduler'; const makeMocks = (target, mocks) => new Map(target.$inject.map((provider) => { return (provider in mocks) ? [provider, mocks[provider]] : [provider, {}]; @@ -121,7 +120,7 @@ suite('Configuration store effects', () => { stateMarbles: 'AB', mocks: { ...mocks, - Clusters: {getCluster: () => _throw({data: 'Error'})} + Clusters: {getCluster: () => throwError({data: 'Error'})} } }); diff --git a/modules/web-console/frontend/app/components/page-configure/store/selectors.js b/modules/web-console/frontend/app/components/page-configure/store/selectors.js index c9df6436ca676..0c70e5991b0b9 100644 --- a/modules/web-console/frontend/app/components/page-configure/store/selectors.js +++ b/modules/web-console/frontend/app/components/page-configure/store/selectors.js @@ -16,13 +16,8 @@ */ import {uniqueName} from 'app/utils/uniqueName'; -import {of} from 'rxjs/observable/of'; -import {empty} from 'rxjs/observable/empty'; -import {combineLatest} from 'rxjs/observable/combineLatest'; -import 'rxjs/add/operator/filter'; -import 'rxjs/add/operator/mergeMap'; -import 'rxjs/add/observable/combineLatest'; -import {Observable} from 'rxjs/Observable'; +import {of, empty, combineLatest, forkJoin, pipe} from 'rxjs'; +import {filter, pluck, map, switchMap, take, distinctUntilChanged, exhaustMap} from 'rxjs/operators'; import {defaultNames} from '../defaultNames'; import {default as Caches} from 'app/services/Caches'; @@ -30,22 +25,22 @@ import {default as Clusters} from 'app/services/Clusters'; import {default as IGFSs} from 'app/services/IGFSs'; import {default as Models} from 'app/services/Models'; -const isDefined = (s) => s.filter((v) => v); +const isDefined = filter((v) => v); -const selectItems = (path) => (s) => s.filter((s) => s).pluck(path).filter((v) => v); +const selectItems = (path) => pipe(filter((s) => s), pluck(path), filter((v) => v)); -const selectValues = (s) => s.map((v) => v && [...v.value.values()]); +const selectValues = map((v) => v && [...v.value.values()]); -export const selectMapItem = (mapPath, key) => (s) => s.pluck(mapPath).map((v) => v && v.get(key)); +export const selectMapItem = (mapPath, key) => pipe(pluck(mapPath), map((v) => v && v.get(key))); -const selectMapItems = (mapPath, keys) => (s) => s.pluck(mapPath).map((v) => v && keys.map((key) => v.get(key))); +const selectMapItems = (mapPath, keys) => pipe(pluck(mapPath), map((v) => v && keys.map((key) => v.get(key)))); -const selectItemToEdit = ({items, itemFactory, defaultName = '', itemID}) => (s) => s.switchMap((item) => { +const selectItemToEdit = ({items, itemFactory, defaultName = '', itemID}) => switchMap((item) => { if (item) return of(Object.assign(itemFactory(), item)); if (itemID === 'new') - return items.take(1).map((items) => Object.assign(itemFactory(), {name: uniqueName(defaultName, items)})); + return items.pipe(take(1), map((items) => Object.assign(itemFactory(), {name: uniqueName(defaultName, items)}))); if (!itemID) return of(null); @@ -54,22 +49,24 @@ const selectItemToEdit = ({items, itemFactory, defaultName = '', itemID}) => (s) }); const currentShortItems = ({changesKey, shortKey}) => (state$) => { - return Observable.combineLatest( - state$.pluck('edit', 'changes', changesKey).let(isDefined).distinctUntilChanged(), - state$.pluck(shortKey, 'value').let(isDefined).distinctUntilChanged() - ) - .map(([{ids = [], changedItems}, shortItems]) => { + return combineLatest( + state$.pipe(pluck('edit', 'changes', changesKey), isDefined, distinctUntilChanged()), + state$.pipe(pluck(shortKey, 'value'), isDefined, distinctUntilChanged()) + ).pipe( + map(([{ids = [], changedItems}, shortItems]) => { if (!ids.length || !shortItems) return []; return ids.map((id) => changedItems.find(({_id}) => _id === id) || shortItems.get(id)); - }) - .map((v) => v.filter((v) => v)); + }), + map((v) => v.filter((v) => v)) + ); }; -const selectNames = (itemIDs, nameAt = 'name') => (items) => items - .pluck('value') - .map((items) => itemIDs.map((id) => items.get(id)[nameAt])); +const selectNames = (itemIDs, nameAt = 'name') => pipe( + pluck('value'), + map((items) => itemIDs.map((id) => items.get(id)[nameAt])) +); export default class ConfigSelectors { static $inject = ['Caches', 'Clusters', 'IGFSs', 'Models']; @@ -95,17 +92,18 @@ export default class ConfigSelectors { * @returns {(state$: Observable) => Observable<{pristine: boolean, value: Map}>} */ this.selectShortModels = () => selectItems('shortModels'); - this.selectShortModelsValue = () => (state$) => state$.let(this.selectShortModels()).let(selectValues); + this.selectShortModelsValue = () => (state$) => state$.pipe(this.selectShortModels(), selectValues); /** * @returns {(state$: Observable) => Observable>} */ - this.selectShortClustersValue = () => (state$) => state$.let(this.selectShortClusters()).let(selectValues); + this.selectShortClustersValue = () => (state$) => state$.pipe(this.selectShortClusters(), selectValues); /** * @returns {(state$: Observable) => Observable>} */ - this.selectClusterNames = (clusterIDs) => (state$) => state$ - .let(this.selectShortClusters()) - .let(selectNames(clusterIDs)); + this.selectClusterNames = (clusterIDs) => (state$) => state$.pipe( + this.selectShortClusters(), + selectNames(clusterIDs) + ); } selectCluster = (id) => selectMapItem('clusters', id); @@ -118,52 +116,56 @@ export default class ConfigSelectors { selectShortCaches = () => selectItems('shortCaches'); - selectShortCachesValue = () => (state$) => state$.let(this.selectShortCaches()).let(selectValues); + selectShortCachesValue = () => (state$) => state$.pipe(this.selectShortCaches(), selectValues); selectShortIGFSs = () => selectItems('shortIgfss'); - selectShortIGFSsValue = () => (state$) => state$.let(this.selectShortIGFSs()).let(selectValues); + selectShortIGFSsValue = () => (state$) => state$.pipe(this.selectShortIGFSs(), selectValues); - selectShortModelsValue = () => (state$) => state$.let(this.selectShortModels()).let(selectValues); + selectShortModelsValue = () => (state$) => state$.pipe(this.selectShortModels(), selectValues); - selectCacheToEdit = (cacheID) => (state$) => state$ - .let(this.selectCache(cacheID)) - .distinctUntilChanged() - .let(selectItemToEdit({ - items: state$.let(this.selectCurrentShortCaches), + selectCacheToEdit = (cacheID) => (state$) => state$.pipe( + this.selectCache(cacheID), + distinctUntilChanged(), + selectItemToEdit({ + items: state$.pipe(this.selectCurrentShortCaches), itemFactory: () => this.Caches.getBlankCache(), defaultName: defaultNames.cache, itemID: cacheID - })); + }) + ); - selectIGFSToEdit = (itemID) => (state$) => state$ - .let(this.selectIGFS(itemID)) - .distinctUntilChanged() - .let(selectItemToEdit({ - items: state$.let(this.selectCurrentShortIGFSs), + selectIGFSToEdit = (itemID) => (state$) => state$.pipe( + this.selectIGFS(itemID), + distinctUntilChanged(), + selectItemToEdit({ + items: state$.pipe(this.selectCurrentShortIGFSs), itemFactory: () => this.IGFSs.getBlankIGFS(), defaultName: defaultNames.igfs, itemID - })); + }) + ); - selectModelToEdit = (itemID) => (state$) => state$ - .let(this.selectModel(itemID)) - .distinctUntilChanged() - .let(selectItemToEdit({ - items: state$.let(this.selectCurrentShortModels), + selectModelToEdit = (itemID) => (state$) => state$.pipe( + this.selectModel(itemID), + distinctUntilChanged(), + selectItemToEdit({ + items: state$.pipe(this.selectCurrentShortModels), itemFactory: () => this.Models.getBlankModel(), itemID - })); + }) + ); - selectClusterToEdit = (clusterID, defaultName = defaultNames.cluster) => (state$) => state$ - .let(this.selectCluster(clusterID)) - .distinctUntilChanged() - .let(selectItemToEdit({ - items: state$.let(this.selectShortClustersValue()), + selectClusterToEdit = (clusterID, defaultName = defaultNames.cluster) => (state$) => state$.pipe( + this.selectCluster(clusterID), + distinctUntilChanged(), + selectItemToEdit({ + items: state$.pipe(this.selectShortClustersValue()), itemFactory: () => this.Clusters.getBlankCluster(), defaultName, itemID: clusterID - })); + }) + ); selectCurrentShortCaches = currentShortItems({changesKey: 'caches', shortKey: 'shortCaches'}); @@ -176,34 +178,33 @@ export default class ConfigSelectors { return of([]); return combineLatest( - state$.let(this.selectCluster(clusterID)).pluck('caches'), - state$.let(this.selectShortCaches()).pluck('value'), + state$.pipe(this.selectCluster(clusterID), pluck('caches')), + state$.pipe(this.selectShortCaches(), pluck('value')), (ids, items) => ids.map((id) => items.get(id)) ); }; selectCompleteClusterConfiguration = ({clusterID, isDemo}) => (state$) => { const hasValues = (array) => !array.some((v) => !v); - return state$.let(this.selectCluster(clusterID)) - .exhaustMap((cluster) => { - if (!cluster) - return of({__isComplete: false}); - - const withSpace = (array) => array.map((c) => ({...c, space: cluster.space})); - - return Observable.forkJoin( - state$.let(selectMapItems('caches', cluster.caches || [])).take(1), - state$.let(selectMapItems('models', cluster.models || [])).take(1), - state$.let(selectMapItems('igfss', cluster.igfss || [])).take(1), - ) - .map(([caches, models, igfss]) => ({ - cluster, - caches, - domains: models, - igfss, - spaces: [{_id: cluster.space, demo: isDemo}], - __isComplete: !!cluster && !(!hasValues(caches) || !hasValues(models) || !hasValues(igfss)) - })); - }); + return state$.pipe( + this.selectCluster(clusterID), + exhaustMap((cluster) => { + if (!cluster) + return of({__isComplete: false}); + + return forkJoin( + state$.pipe(selectMapItems('caches', cluster.caches || []), take(1)), + state$.pipe(selectMapItems('models', cluster.models || []), take(1)), + state$.pipe(selectMapItems('igfss', cluster.igfss || []), take(1)), + ).pipe(map(([caches, models, igfss]) => ({ + cluster, + caches, + domains: models, + igfss, + spaces: [{_id: cluster.space, demo: isDemo}], + __isComplete: !!cluster && !(!hasValues(caches) || !hasValues(models) || !hasValues(igfss)) + }))); + }) + ); }; } diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts index 3ac3177a7e2ee..78274e67674a5 100644 --- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts +++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts @@ -18,15 +18,8 @@ import _ from 'lodash'; import {nonEmpty, nonNil} from 'app/utils/lodashMixins'; import id8 from 'app/utils/id8'; -import 'rxjs/add/operator/mergeMap'; -import 'rxjs/add/operator/merge'; -import 'rxjs/add/operator/switchMap'; -import 'rxjs/add/operator/exhaustMap'; -import 'rxjs/add/operator/distinctUntilChanged'; - -import { fromPromise } from 'rxjs/observable/fromPromise'; -import { timer } from 'rxjs/observable/timer'; -import { defer } from 'rxjs/observable/defer'; +import {timer, merge, defer, from} from 'rxjs'; +import {mergeMap, tap, switchMap, exhaustMap, take} from 'rxjs/operators'; import {CSV} from 'app/services/CSV'; @@ -930,27 +923,28 @@ export class NotebookCtrl { }; const _startWatch = () => { - const awaitClusters$ = fromPromise( + const awaitClusters$ = from( agentMgr.startClusterWatch('Leave Queries', 'default-state')); const finishLoading$ = defer(() => { if (!$root.IgniteDemoMode) Loading.finish('sqlLoading'); - }).take(1); + }).pipe(take(1)); const refreshCaches = (period) => { - return timer(0, period).exhaustMap(() => _refreshCaches()).merge(finishLoading$); + return merge(timer(0, period).pipe(exhaustMap(() => _refreshCaches())), finishLoading$); }; - this.refresh$ = awaitClusters$ - .mergeMap(() => agentMgr.currentCluster$) - .do(() => Loading.start('sqlLoading')) - .do(() => { + this.refresh$ = awaitClusters$.pipe( + mergeMap(() => agentMgr.currentCluster$), + tap(() => Loading.start('sqlLoading')), + tap(() => { _.forEach($scope.notebook.paragraphs, (paragraph) => { paragraph.reset($interval); }); - }) - .switchMap(() => refreshCaches(5000)) + }), + switchMap(() => refreshCaches(5000)) + ) .subscribe(); }; diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index 6450cd67da230..0c0d9b69f4e43 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -18,11 +18,8 @@ import _ from 'lodash'; import {nonEmpty, nonNil} from 'app/utils/lodashMixins'; -import {BehaviorSubject} from 'rxjs/BehaviorSubject'; -import 'rxjs/add/operator/first'; -import 'rxjs/add/operator/partition'; -import 'rxjs/add/operator/takeUntil'; -import 'rxjs/add/operator/pluck'; +import {BehaviorSubject} from 'rxjs'; +import {first, pluck, tap, distinctUntilChanged, map, filter} from 'rxjs/operators'; import io from 'socket.io-client'; @@ -185,14 +182,16 @@ export default class AgentManager { let prevCluster; - this.currentCluster$ = this.connectionSbj - .distinctUntilChanged(({ cluster }) => prevCluster === cluster) - .do(({ cluster }) => prevCluster = cluster); + this.currentCluster$ = this.connectionSbj.pipe( + distinctUntilChanged(({ cluster }) => prevCluster === cluster), + tap(({ cluster }) => prevCluster = cluster) + ); - this.clusterIsActive$ = this.connectionSbj - .map(({ cluster }) => cluster) - .filter((cluster) => Boolean(cluster)) - .pluck('active'); + this.clusterIsActive$ = this.connectionSbj.pipe( + map(({ cluster }) => cluster), + filter((cluster) => Boolean(cluster)), + pluck('active') + ); if (!this.isDemoMode()) { this.connectionSbj.subscribe({ @@ -526,7 +525,7 @@ export default class AgentManager { if (this.isDemoMode()) return Promise.resolve(this._executeOnActiveCluster({}, {}, event, params)); - return this.connectionSbj.first().toPromise() + return this.connectionSbj.pipe(first()).toPromise() .then(({cluster}) => { if (_.isNil(cluster)) throw new Error('Failed to execute request on cluster.'); diff --git a/modules/web-console/frontend/app/modules/user/User.service.js b/modules/web-console/frontend/app/modules/user/User.service.js index 3515c1b96821b..da84dcc7e9575 100644 --- a/modules/web-console/frontend/app/modules/user/User.service.js +++ b/modules/web-console/frontend/app/modules/user/User.service.js @@ -15,7 +15,7 @@ * limitations under the License. */ -import {ReplaySubject} from 'rxjs/ReplaySubject'; +import {ReplaySubject} from 'rxjs'; /** * @typedef User diff --git a/modules/web-console/frontend/app/services/Clusters.js b/modules/web-console/frontend/app/services/Clusters.js index e0a2ec7ca40b4..5649f0af6aa42 100644 --- a/modules/web-console/frontend/app/services/Clusters.js +++ b/modules/web-console/frontend/app/services/Clusters.js @@ -16,8 +16,7 @@ */ import get from 'lodash/get'; -import {Observable} from 'rxjs/Observable'; -import 'rxjs/add/observable/fromPromise'; +import {from} from 'rxjs'; import ObjectID from 'bson-objectid/objectid'; import {uniqueName} from 'app/utils/uniqueName'; import omit from 'lodash/fp/omit'; @@ -117,7 +116,7 @@ export default class Clusters { } getClustersOverview$() { - return Observable.fromPromise(this.getClustersOverview()); + return from(this.getClustersOverview()); } saveCluster(cluster) { @@ -125,7 +124,7 @@ export default class Clusters { } saveCluster$(cluster) { - return Observable.fromPromise(this.saveCluster(cluster)); + return from(this.saveCluster(cluster)); } removeCluster(cluster) { @@ -133,7 +132,7 @@ export default class Clusters { } removeCluster$(cluster) { - return Observable.fromPromise(this.removeCluster(cluster)); + return from(this.removeCluster(cluster)); } saveBasic(changedItems) { diff --git a/modules/web-console/frontend/app/services/Version.service.js b/modules/web-console/frontend/app/services/Version.service.js index bbd03cb92f1d8..cbaeff1c20826 100644 --- a/modules/web-console/frontend/app/services/Version.service.js +++ b/modules/web-console/frontend/app/services/Version.service.js @@ -15,7 +15,7 @@ * limitations under the License. */ -import { BehaviorSubject } from 'rxjs/BehaviorSubject'; +import { BehaviorSubject } from 'rxjs'; import _ from 'lodash'; /** diff --git a/modules/web-console/frontend/app/services/store.ts b/modules/web-console/frontend/app/services/store.ts index 19dde69e22ebd..4498d1e0ecd66 100644 --- a/modules/web-console/frontend/app/services/store.ts +++ b/modules/web-console/frontend/app/services/store.ts @@ -15,10 +15,8 @@ * limitations under the License. */ -import {BehaviorSubject} from 'rxjs/BehaviorSubject'; -import {Subject} from 'rxjs/Subject'; +import {BehaviorSubject, Subject, merge} from 'rxjs'; import {scan, tap} from 'rxjs/operators'; -import {merge} from 'rxjs/observable/merge'; interface Reducer { (state: State, action: Actions): State @@ -27,9 +25,9 @@ interface Reducer { export class Store { static $inject = ['$injector']; - actions$: Subject - state$: BehaviorSubject - private _reducers: Array> + actions$: Subject; + state$: BehaviorSubject; + private _reducers: Array>; constructor(private $injector: ng.auto.IInjectorService) { this.$injector = $injector; diff --git a/modules/web-console/frontend/app/store/ofType.ts b/modules/web-console/frontend/app/store/ofType.ts index 3d3bd8e3d73e2..220de5f7884f3 100644 --- a/modules/web-console/frontend/app/store/ofType.ts +++ b/modules/web-console/frontend/app/store/ofType.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import {OperatorFunction} from 'rxjs/interfaces'; +import {OperatorFunction} from 'rxjs'; import {filter} from 'rxjs/operators'; type Action = {type: string}; diff --git a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js index 1beb418469ba8..b751dc3e5e235 100644 --- a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js +++ b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js @@ -16,15 +16,8 @@ */ import _ from 'lodash'; - -import {Observable} from 'rxjs/Observable'; -import {Subject} from 'rxjs/Subject'; -import 'rxjs/add/observable/race'; -import 'rxjs/add/operator/filter'; -import 'rxjs/add/operator/map'; -import 'rxjs/add/operator/pluck'; -import 'rxjs/add/operator/take'; -import 'rxjs/add/operator/toPromise'; +import {Subject, race} from 'rxjs'; +import {filter, take, pluck, map} from 'rxjs/operators'; /** * Simple implementation of workers pool. @@ -112,11 +105,11 @@ export default class SimpleWorkerPool { this._run(); - return Observable.race( - this.messages$.filter((e) => e.tid === tid).take(1).pluck('m', 'data'), - this.errors$.filter((e) => e.tid === tid).take(1).map((e) => { + return race( + this.messages$.pipe(filter((e) => e.tid === tid), take(1), pluck('m', 'data')), + this.errors$.pipe(filter((e) => e.tid === tid), take(1), map((e) => { throw e.e; })) - .take(1).toPromise(); + ).pipe(take(1)).toPromise(); } } diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json index 3c4473f2c3035..aa6e037530e04 100644 --- a/modules/web-console/frontend/package.json +++ b/modules/web-console/frontend/package.json @@ -30,7 +30,7 @@ "@babel/plugin-transform-parameters": "7.0.0", "@uirouter/angularjs": "1.0.20", "@uirouter/core": "5.0.19", - "@uirouter/rx": "0.4.1", + "@uirouter/rx": "0.5.0", "@uirouter/visualizer": "4.0.2", "angular": "1.7.4", "angular-acl": "0.1.10", @@ -69,7 +69,7 @@ "pako": "1.0.6", "resize-observer-polyfill": "1.5.0", "roboto-font": "0.1.0", - "rxjs": "5.5.11", + "rxjs": "6.3.3", "socket.io-client": "2.1.1", "tf-metatags": "2.0.0" }, From 8ce6c5ea683e57b20b9492daf80ed793e1d260d7 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Fri, 28 Dec 2018 13:52:50 +0700 Subject: [PATCH 292/403] IGNITE-10839 Web Console: Added optional email confirmation. Co-authored-by: Andrey Novikov Co-authored-by: Ilya Borisov (cherry picked from commit cdf2d867fecef4459edb0f128c017cbb7b21ba07) --- modules/web-console/assembly/README.txt | 25 +++--- modules/web-console/backend/app/mongo.js | 20 ++++- modules/web-console/backend/app/schemas.js | 5 +- modules/web-console/backend/app/settings.js | 14 ++++ .../backend/config/settings.json.sample | 5 ++ .../MissingConfirmRegistrationException.js | 34 ++++++++ modules/web-console/backend/errors/index.js | 2 + .../web-console/backend/middlewares/api.js | 3 + modules/web-console/backend/routes/public.js | 53 ++++++++---- modules/web-console/backend/services/auth.js | 83 +++++++++++++++++-- modules/web-console/backend/services/mails.js | 18 ++++ modules/web-console/backend/services/users.js | 32 +++++-- modules/web-console/frontend/app/app.js | 5 +- .../list-of-registered-users/categories.js | 1 + .../list-of-registered-users/column-defs.js | 1 + .../app/components/page-signin/component.ts | 5 +- .../app/components/page-signin/controller.ts | 10 ++- .../app/components/page-signin/run.ts | 11 ++- .../app/components/page-signin/template.pug | 4 +- .../page-signup-confirmation/component.ts | 28 +++++++ .../page-signup-confirmation/controller.ts | 42 ++++++++++ .../page-signup-confirmation/index.ts | 23 +++++ .../page-signup-confirmation/state.ts | 48 +++++++++++ .../page-signup-confirmation/style.scss | 22 +++++ .../page-signup-confirmation/template.tpl.pug | 24 ++++++ .../app/components/page-signup/controller.ts | 7 ++ .../frontend/app/modules/user/Auth.service.ts | 14 +++- .../user/emailConfirmationInterceptor.ts | 37 +++++++++ .../frontend/app/modules/user/user.module.js | 27 +++--- 29 files changed, 540 insertions(+), 63 deletions(-) create mode 100644 modules/web-console/backend/errors/MissingConfirmRegistrationException.js create mode 100644 modules/web-console/frontend/app/components/page-signup-confirmation/component.ts create mode 100644 modules/web-console/frontend/app/components/page-signup-confirmation/controller.ts create mode 100644 modules/web-console/frontend/app/components/page-signup-confirmation/index.ts create mode 100644 modules/web-console/frontend/app/components/page-signup-confirmation/state.ts create mode 100644 modules/web-console/frontend/app/components/page-signup-confirmation/style.scss create mode 100644 modules/web-console/frontend/app/components/page-signup-confirmation/template.tpl.pug create mode 100644 modules/web-console/frontend/app/modules/user/emailConfirmationInterceptor.ts diff --git a/modules/web-console/assembly/README.txt b/modules/web-console/assembly/README.txt index 2656aca95fd52..e88e34570cb13 100644 --- a/modules/web-console/assembly/README.txt +++ b/modules/web-console/assembly/README.txt @@ -41,21 +41,24 @@ Technical details On Windows: `ignite-web-console-win.exe --server:port 3000` All available parameters with defaults: - Web Console host: --server:host 0.0.0.0 - Web Console port: --server:port 80 + Web Console host: --server:host 0.0.0.0 + Web Console port: --server:port 80 - Enable HTTPS: --server:ssl false + Enable HTTPS: --server:ssl false + Disable self registration: --server:disable:signup false - Disable self registration: --server:disable:signup false + MongoDB URL: --mongodb:url mongodb://localhost/console - MongoDB URL: --mongodb:url mongodb://localhost/console + Enable account activation: --activation:enabled false + Activation timeout(milliseconds): --activation:timeout 1800000 + Activation send email throttle (milliseconds): --activation:sendTimeout 180000 - Mail service: --mail:service "gmail" - Signature text: --mail:sign "Kind regards, Apache Ignite Team" - Greeting text: --mail:greeting "Apache Ignite Web Console" - Mail FROM: --mail:from "Apache Ignite Web Console " - User to send e-mail: --mail:auth:user "someusername@somecompany.somedomain" - E-mail service password: --mail:auth:pass "" + Mail service: --mail:service "gmail" + Signature text: --mail:sign "Kind regards, Apache Ignite Team" + Greeting text: --mail:greeting "Apache Ignite Web Console" + Mail FROM: --mail:from "Apache Ignite Web Console " + User to send e-mail: --mail:auth:user "someusername@somecompany.somedomain" + E-mail service password: --mail:auth:pass "" SSL options has no default values: --server:key "path to file with server.key" diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js index 6a843dbdd15e7..57171aceadda2 100644 --- a/modules/web-console/backend/app/mongo.js +++ b/modules/web-console/backend/app/mongo.js @@ -128,7 +128,8 @@ module.exports.factory = function(settings, mongoose, schemas) { admin: true, token: 'ruQvlWff09zqoVYyh6WJ', attempts: 0, - resetPasswordToken: 'O2GWgOkKkhqpDcxjYnSP' + resetPasswordToken: 'O2GWgOkKkhqpDcxjYnSP', + activated: true }), mongo.Space.create({ _id: '59fc0c26e145c32be0f83b34', @@ -146,5 +147,22 @@ module.exports.factory = function(settings, mongoose, schemas) { return mongo; }); + }) + .then((mongo) => { + if (settings.activation.enabled) { + return mongo.Account.find({ + $or: [{activated: false}, {activated: {$exists: false}}], + activationToken: {$exists: false} + }, '_id').lean().exec() + .then((accounts) => { + const conditions = _.map(accounts, (account) => ({session: {$regex: `"${account._id}"`}})); + + return mongoose.connection.db.collection('sessions').deleteMany({$or: conditions}); + }) + .then(() => mongo) + .catch(() => mongo); + } + + return mongo; }); }; diff --git a/modules/web-console/backend/app/schemas.js b/modules/web-console/backend/app/schemas.js index 69d4a4cce81d6..95c3f5b1b2503 100644 --- a/modules/web-console/backend/app/schemas.js +++ b/modules/web-console/backend/app/schemas.js @@ -46,7 +46,10 @@ module.exports.factory = function(mongoose) { lastActivity: Date, admin: Boolean, token: String, - resetPasswordToken: String + resetPasswordToken: String, + activated: {type: Boolean, default: false}, + activationSentAt: Date, + activationToken: String }); // Install passport plugin. diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js index 9c4d3c3fb380d..0079788615473 100644 --- a/modules/web-console/backend/app/settings.js +++ b/modules/web-console/backend/app/settings.js @@ -18,6 +18,7 @@ 'use strict'; const fs = require('fs'); +const _ = require('lodash'); // Fire me up! @@ -61,6 +62,14 @@ module.exports = { return v === 'true' || v === true; }; + let activationEnabled = _isTrue('activation:enabled'); + + if (activationEnabled && _.isEmpty(mail)) { + activationEnabled = false; + + console.warn('Mail server settings are required for account confirmation!'); + } + const settings = { agent: { dists: nconf.get('agent:dists') || dfltAgentDists @@ -72,6 +81,11 @@ module.exports = { disableSignup: _isTrue('server:disable:signup') }, mail, + activation: { + enabled: activationEnabled, + timeout: nconf.get('activation:timeout') || 1800000, + sendTimeout: nconf.get('activation:sendTimeout') || 180000 + }, mongoUrl: nconf.get('mongodb:url') || 'mongodb://127.0.0.1/console', cookieTTL: 3600000 * 24 * 30, sessionSecret: nconf.get('server:sessionSecret') || 'keyboard cat', diff --git a/modules/web-console/backend/config/settings.json.sample b/modules/web-console/backend/config/settings.json.sample index c16ba265bd655..02bc3270b8402 100644 --- a/modules/web-console/backend/config/settings.json.sample +++ b/modules/web-console/backend/config/settings.json.sample @@ -18,6 +18,11 @@ "mongodb": { "url": "mongodb://localhost/console" }, + "activation": { + "enabled": false, + "timeout": 1800000, + "sendTimeout": 180000 + }, "mail": { "service": "gmail", "from": "Some Company Web Console ", diff --git a/modules/web-console/backend/errors/MissingConfirmRegistrationException.js b/modules/web-console/backend/errors/MissingConfirmRegistrationException.js new file mode 100644 index 0000000000000..a094a679bf33f --- /dev/null +++ b/modules/web-console/backend/errors/MissingConfirmRegistrationException.js @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const IllegalAccessError = require('./IllegalAccessError'); + +class MissingConfirmRegistrationException extends IllegalAccessError { + constructor(email) { + super('User account email not activated'); + + this.data = { + errorCode: 10104, + message: this.message, + email + }; + } +} + +module.exports = MissingConfirmRegistrationException; diff --git a/modules/web-console/backend/errors/index.js b/modules/web-console/backend/errors/index.js index 2fadc120ab08d..cb8f043503071 100644 --- a/modules/web-console/backend/errors/index.js +++ b/modules/web-console/backend/errors/index.js @@ -24,6 +24,7 @@ const IllegalArgumentException = require('./IllegalArgumentException'); const IllegalAccessError = require('./IllegalAccessError'); const DuplicateKeyException = require('./DuplicateKeyException'); const ServerErrorException = require('./ServerErrorException'); +const MissingConfirmRegistrationException = require('./MissingConfirmRegistrationException'); const MissingResourceException = require('./MissingResourceException'); const AuthFailedException = require('./AuthFailedException'); @@ -35,6 +36,7 @@ module.exports = { IllegalArgumentException, DuplicateKeyException, ServerErrorException, + MissingConfirmRegistrationException, MissingResourceException, AuthFailedException }) diff --git a/modules/web-console/backend/middlewares/api.js b/modules/web-console/backend/middlewares/api.js index 27e130d4e8e38..d4d832b5a650f 100644 --- a/modules/web-console/backend/middlewares/api.js +++ b/modules/web-console/backend/middlewares/api.js @@ -37,6 +37,9 @@ module.exports.factory = () => { if (_.includes(['MongoError', 'MongooseError'], err.name)) return res.status(500).send(err.message); + if (_.isObject(err.data)) + return res.status(err.httpCode || err.code || 500).json(err.data); + res.status(err.httpCode || err.code || 500).send(err.message); }, diff --git a/modules/web-console/backend/routes/public.js b/modules/web-console/backend/routes/public.js index 8aa3b4338db4b..d290b53520553 100644 --- a/modules/web-console/backend/routes/public.js +++ b/modules/web-console/backend/routes/public.js @@ -25,17 +25,18 @@ const passport = require('passport'); module.exports = { implements: 'routes/public', - inject: ['mongo', 'services/mails', 'services/users', 'services/auth'] + inject: ['mongo', 'settings', 'services/users', 'services/auth', 'errors'] }; -/** +/** * @param mongo - * @param mailsService + * @param settings * @param {UsersService} usersService * @param {AuthService} authService + * @param errors * @returns {Promise} */ -module.exports.factory = function(mongo, mailsService, usersService, authService) { +module.exports.factory = function(mongo, settings, usersService, authService, errors) { return new Promise((factoryResolve) => { const router = new express.Router(); @@ -76,14 +77,34 @@ module.exports.factory = function(mongo, mailsService, usersService, authService router.post('/signin', (req, res, next) => { passport.authenticate('local', (errAuth, user) => { if (errAuth) - return res.status(401).send(errAuth.message); + return res.api.error(new errors.AuthFailedException(errAuth.message)); if (!user) - return res.status(401).send('Invalid email or password'); + return res.api.error(new errors.AuthFailedException('Invalid email or password')); - req.logIn(user, {}, (errLogIn) => { + if (settings.activation.enabled) { + const activationToken = req.body.activationToken; + + const errToken = authService.validateActivationToken(user, activationToken); + + if (errToken) + return res.api.error(errToken); + + if (authService.isActivationTokenExpired(user, activationToken)) { + authService.resetActivationToken(req.origin(), user.email) + .catch((ignored) => { + // No-op. + }); + + return res.api.error(new errors.AuthFailedException('This activation link was expired. We resend a new one. Please open the most recent email and click on the activation link.')); + } + + user.activated = true; + } + + return req.logIn(user, {}, (errLogIn) => { if (errLogIn) - return res.status(401).send(errLogIn.message); + return res.api.error(new errors.AuthFailedException(errLogIn.message)); return res.sendStatus(200); }); @@ -103,10 +124,8 @@ module.exports.factory = function(mongo, mailsService, usersService, authService * Send e-mail to user with reset token. */ router.post('/password/forgot', (req, res) => { - authService.resetPasswordToken(req.body.email) - .then((user) => mailsService.emailUserResetLink(req.origin(), user)) - .then(() => 'An email has been sent with further instructions.') - .then(res.api.ok) + authService.resetPasswordToken(req.origin(), req.body.email) + .then(() => res.api.ok('An email has been sent with further instructions.')) .catch(res.api.error); }); @@ -116,8 +135,7 @@ module.exports.factory = function(mongo, mailsService, usersService, authService router.post('/password/reset', (req, res) => { const {token, password} = req.body; - authService.resetPasswordByToken(token, password) - .then((user) => mailsService.emailPasswordChanged(req.origin(), user)) + authService.resetPasswordByToken(req.origin(), token, password) .then(res.api.ok) .catch(res.api.error); }); @@ -131,6 +149,13 @@ module.exports.factory = function(mongo, mailsService, usersService, authService .catch(res.api.error); }); + /* Send e-mail to user with account confirmation token. */ + router.post('/activation/resend', (req, res) => { + authService.resetActivationToken(req.origin(), req.body.email) + .then(() => res.api.ok('An email has been sent with further instructions.')) + .catch(res.api.error); + }); + factoryResolve(router); }); }; diff --git a/modules/web-console/backend/services/auth.js b/modules/web-console/backend/services/auth.js index 986ed9555d0df..467995724e1d8 100644 --- a/modules/web-console/backend/services/auth.js +++ b/modules/web-console/backend/services/auth.js @@ -19,9 +19,11 @@ // Fire me up! +const _ = require('lodash'); + module.exports = { implements: 'services/auth', - inject: ['mongo', 'settings', 'errors', 'services/utils'] + inject: ['mongo', 'settings', 'errors', 'services/utils', 'services/mails'] }; /** @@ -29,37 +31,44 @@ module.exports = { * @param settings * @param errors * @param {UtilsService} utilsService + * @param {MailsService} mailsService * @returns {AuthService} */ -module.exports.factory = (mongo, settings, errors, utilsService) => { +module.exports.factory = (mongo, settings, errors, utilsService, mailsService) => { class AuthService { /** * Reset password reset token for user. * + * @param host Web Console host. * @param email - user email * @returns {Promise.} - that resolves account found by email with new reset password token. */ - static resetPasswordToken(email) { + static resetPasswordToken(host, email) { return mongo.Account.findOne({email}).exec() .then((user) => { if (!user) throw new errors.MissingResourceException('Account with that email address does not exists!'); + if (settings.activation.enabled && !user.activated) + throw new errors.MissingConfirmRegistrationException(user.email); + user.resetPasswordToken = utilsService.randomString(settings.tokenLength); return user.save(); - }); + }) + .then((user) => mailsService.emailUserResetLink(host, user)); } /** * Reset password by reset token. * + * @param host Web Console host. * @param {string} token - reset token * @param {string} newPassword - new password * @returns {Promise.} - that resolves account with new password */ - static resetPasswordByToken(token, newPassword) { + static resetPasswordByToken(host, token, newPassword) { return mongo.Account.findOne({resetPasswordToken: token}).exec() .then((user) => { if (!user) @@ -75,7 +84,8 @@ module.exports.factory = (mongo, settings, errors, utilsService) => { resolve(_user.save()); }); }); - }); + }) + .then((user) => mailsService.emailPasswordChanged(host, user)); } /** @@ -93,6 +103,67 @@ module.exports.factory = (mongo, settings, errors, utilsService) => { return {token, email: user.email}; }); } + + /** + * Validate activationToken token. + * + * @param {mongo.Account} user - User object. + * @param {string} activationToken - activate account token + * @return {Error} If token is invalid. + */ + static validateActivationToken(user, activationToken) { + if (user.activated) { + if (user.activationToken !== activationToken) + return new errors.AuthFailedException('Invalid email or password!'); + } + else { + if (_.isEmpty(activationToken)) + return new errors.MissingConfirmRegistrationException(user.email); + + if (user.activationToken !== activationToken) + return new errors.AuthFailedException('This activation token isn\'t valid.'); + } + } + + /** + * Check if activation token expired. + * + * @param {mongo.Account} user - User object. + * @param {string} activationToken - activate account token + * @return {boolean} If token was already expired. + */ + static isActivationTokenExpired(user, activationToken) { + return !user.activated && + new Date().getTime() - user.activationSentAt.getTime() >= settings.activation.timeout; + } + + /** + * Reset password reset token for user. + * + * @param host Web Console host. + * @param email - user email. + * @returns {Promise}. + */ + static resetActivationToken(host, email) { + return mongo.Account.findOne({email}).exec() + .then((user) => { + if (!user) + throw new errors.MissingResourceException('Account with that email address does not exists!'); + + if (!settings.activation.enabled) + throw new errors.IllegalAccessError('Activation was not enabled!'); + + if (user.activationSentAt && + new Date().getTime() - user.activationSentAt.getTime() < settings.activation.sendTimeout) + throw new errors.IllegalAccessError('Too Many Activation Attempts!'); + + user.activationToken = utilsService.randomString(settings.tokenLength); + user.activationSentAt = new Date(); + + return user.save(); + }) + .then((user) => mailsService.emailUserActivation(host, user)); + } } return AuthService; diff --git a/modules/web-console/backend/services/mails.js b/modules/web-console/backend/services/mails.js index 75da128ad9dca..183fbe1086e30 100644 --- a/modules/web-console/backend/services/mails.js +++ b/modules/web-console/backend/services/mails.js @@ -108,6 +108,24 @@ module.exports.factory = (settings) => { /** * Send email to user for password reset. + * + * @param host + * @param user + */ + emailUserActivation(host, user) { + const activationLink = `${host}/signin?activationToken=${user.activationToken}`; + + return this.send(user, `Confirm your account on ${settings.mail.greeting}`, + `Hello ${user.firstName} ${user.lastName}!

` + + `You are receiving this email because you have signed up to use ${settings.mail.greeting}.

` + + 'Please click on the following link, or paste this into your browser to activate your account:

' + + `${activationLink}`, + 'Failed to send email with confirm account link!'); + } + + /** + * Send email to user for password reset. + * * @param host * @param user */ diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js index 79578c5b45995..ed844db61e14e 100644 --- a/modules/web-console/backend/services/users.js +++ b/modules/web-console/backend/services/users.js @@ -53,6 +53,8 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService, user.admin = cnt === 0; user.registered = new Date(); user.token = utilsService.randomString(settings.tokenLength); + user.resetPasswordToken = utilsService.randomString(settings.tokenLength); + user.activated = false; if (settings.server.disableSignup && !user.admin && !createdByAdmin) throw new errors.ServerErrorException('Sign-up is not allowed. Ask your Web Console administrator to create account for you.'); @@ -73,15 +75,27 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService, }); }) .then((registered) => { - registered.resetPasswordToken = utilsService.randomString(settings.tokenLength); + return mongo.Space.create({name: 'Personal space', owner: registered._id}) + .then(() => registered) + }) + .then((registered) => { + if (settings.activation.enabled) { + registered.activationToken = utilsService.randomString(settings.tokenLength); + registered.activationSentAt = new Date(); - return registered.save() - .then(() => mongo.Space.create({name: 'Personal space', owner: registered._id})) - .then(() => { - mailsService.emailUserSignUp(host, registered, createdByAdmin); + if (!createdByAdmin) { + return registered.save() + .then(() => { + mailsService.emailUserActivation(host, registered); - return registered; - }); + throw new errors.MissingConfirmRegistrationException(registered.email); + }); + } + } + + mailsService.emailUserSignUp(host, registered, createdByAdmin); + + return registered; }); } @@ -93,6 +107,9 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService, */ static save(changed) { delete changed.admin; + delete changed.activated; + delete changed.activationSentAt; + delete changed.activationToken; return mongo.Account.findById(changed._id).exec() .then((user) => { @@ -157,6 +174,7 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService, country: 1, lastLogin: 1, lastActivity: 1, + activated: 1, spaces: { $filter: { input: '$spaces', diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index fd75adee0ecdd..94f97042f84f5 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -161,6 +161,7 @@ import pageForgotPassword from './components/page-forgot-password'; import formSignup from './components/form-signup'; import sidebar from './components/web-console-sidebar'; import permanentNotifications from './components/permanent-notifications'; +import signupConfirmation from './components/page-signup-confirmation'; import igniteServices from './services'; @@ -268,7 +269,9 @@ export default angular.module('ignite-console', [ formSignup.name, timedRedirection.name, sidebar.name, - permanentNotifications.name + permanentNotifications.name, + timedRedirection.name, + signupConfirmation.name ]) .service('$exceptionHandler', $exceptionHandler) // Directives. diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/categories.js b/modules/web-console/frontend/app/components/list-of-registered-users/categories.js index e73dd17c0f8f9..aa70863f5e536 100644 --- a/modules/web-console/frontend/app/components/list-of-registered-users/categories.js +++ b/modules/web-console/frontend/app/components/list-of-registered-users/categories.js @@ -19,6 +19,7 @@ export default [ {name: 'Actions', visible: false, enableHiding: false}, {name: 'User', visible: true, enableHiding: false}, {name: 'Email', visible: true, enableHiding: true}, + {name: 'Activated', visible: false, enableHiding: true}, {name: 'Company', visible: true, enableHiding: true}, {name: 'Country', visible: true, enableHiding: true}, {name: 'Last login', visible: false, enableHiding: true}, diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js index 2094f0c7909c7..b1c71f78448b1 100644 --- a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js +++ b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js @@ -32,6 +32,7 @@ const VALUE_WITH_TITLE = '